From df9ce757a77e4bf49084fecb5972b6292fc02491 Mon Sep 17 00:00:00 2001 From: xmy Date: Thu, 9 Nov 2023 16:44:31 +0800 Subject: [PATCH] [CH] enable replaceSortAggWithHashAgg --- .../execution/GlutenClickHouseFileFormatSuite.scala | 5 ++--- .../io/glutenproject/backendsapi/velox/VeloxBackend.scala | 2 -- .../io/glutenproject/backendsapi/BackendSettingsApi.scala | 2 +- .../src/main/scala/io/glutenproject/utils/FallbackUtil.scala | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala index b185fe8e5f800..b9200ba03d67f 100644 --- a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala +++ b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseFileFormatSuite.scala @@ -196,7 +196,7 @@ class GlutenClickHouseFileFormatSuite ) } - test("read data from csv file format witsh agg") { + test("read data from csv file format with agg") { val filePath = basePath + "/csv_test_agg.csv" val csvFileFormat = "csv" val sql = @@ -214,8 +214,7 @@ class GlutenClickHouseFileFormatSuite case f: FileSourceScanExecTransformer => f } assert(csvFileScan.size == 1) - }, - noFallBack = false + } ) } diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala index 40d42b90873ec..5fe04bd1db84b 100644 --- a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala @@ -292,8 +292,6 @@ object BackendSettings extends BackendSettingsApi { override def removeHashColumnFromColumnarShuffleExchangeExec(): Boolean = true override def rescaleDecimalLiteral(): Boolean = true - override def replaceSortAggWithHashAgg: Boolean = GlutenConfig.getConf.forceToUseHashAgg - /** Get the config prefix for each backend */ override def getBackendConfigPrefix(): String = GlutenConfig.GLUTEN_CONFIG_PREFIX + VeloxBackend.BACKEND_NAME diff --git a/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala b/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala index fcd1bbfe84533..4ba0d1bc26e1b 100644 --- a/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala +++ b/gluten-core/src/main/scala/io/glutenproject/backendsapi/BackendSettingsApi.scala @@ -82,7 +82,7 @@ trait BackendSettingsApi { * Whether to replace sort agg with hash agg., e.g., sort agg will be used in spark's planning for * string type input. */ - def replaceSortAggWithHashAgg: Boolean = false + def replaceSortAggWithHashAgg: Boolean = GlutenConfig.getConf.forceToUseHashAgg /** Get the config prefix for each backend */ def getBackendConfigPrefix: String diff --git a/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala b/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala index 40c0786599f26..a98dc7ffdb4fd 100644 --- a/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala +++ b/gluten-core/src/main/scala/io/glutenproject/utils/FallbackUtil.scala @@ -78,9 +78,9 @@ object FallbackUtil extends Logging with AdaptiveSparkPlanHelper { } } - if (!fallbackOperator.isEmpty) { + if (fallbackOperator.nonEmpty) { fallbackOperator.foreach(operator => log.info(s"gluten fallback operator:{$operator}")) } - return fallbackOperator.nonEmpty + fallbackOperator.nonEmpty } }