From 821a5145161c6d6b08bda293efac70bae8721019 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 23 Aug 2024 11:50:01 +0800 Subject: [PATCH] fixup --- .../scala/org/apache/gluten/execution/VeloxTPCHSuite.scala | 1 + .../src/main/scala/org/apache/gluten/GlutenPlugin.scala | 2 +- .../main/scala/org/apache/spark/util/SparkResourceUtil.scala | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 0e94c242c1db..b87f8c05eaa0 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -48,6 +48,7 @@ abstract class VeloxTPCHTableSupport extends VeloxWholeStageTransformerSuite { .set("spark.memory.offHeap.size", "2g") .set("spark.unsafe.exceptionOnMemoryLeak", "true") .set("spark.sql.autoBroadcastJoinThreshold", "-1") + .set("spark.gluten.sql.columnar.backend.velox.glogSeverityLevel", "0") // TODO Should enable this after fix the issue of native plan detail occasional disappearance // .set("spark.gluten.sql.injectNativePlanStringToExplain", "true") } diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 5d25de3aeb0f..5e0c1758d0b4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -181,7 +181,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { 1024 * 1024 * 1024 } - val overheadSize : Long = SparkResourceUtil.getMemoryOverheadSize(conf) + val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf) conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString) // If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkResourceUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkResourceUtil.scala index d298ab481a0c..890ea31b6f1b 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkResourceUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkResourceUtil.scala @@ -87,9 +87,9 @@ object SparkResourceUtil extends Logging { val overheadMib = conf.get(EXECUTOR_MEMORY_OVERHEAD).getOrElse { val executorMemMib = conf.get(EXECUTOR_MEMORY) val factor = - conf.getDouble("spark.executor.memoryOverheadFactor", 0.1D) + conf.getDouble("spark.executor.memoryOverheadFactor", 0.1d) val minMib = conf.getLong("spark.executor.minMemoryOverhead", 384L) - (executorMemMib * factor).toLong max minMib + (executorMemMib * factor).toLong.max(minMib) } ByteUnit.MiB.toBytes(overheadMib) }