From 2a2d2bdeb8eb5d083d3b0260fa84dc93e2f71151 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 26 Aug 2024 16:04:59 +0800 Subject: [PATCH] [VL] Following #6988, move a warning from core to Velox backend --- .../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 +++++++ .../src/main/scala/org/apache/gluten/GlutenPlugin.scala | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 2cfc4e9a9099..56b439a37c2c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -42,6 +42,13 @@ class VeloxListenerApi extends ListenerApi with Logging { override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = { val conf = pc.conf() + // FIXME: The following is a workaround. Remove once the causes are fixed. + conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString) + logWarning( + "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" + + " temporary solution to avoid OOM by Velox's global memory pools." + + " See GLUTEN-6960 for more information.") + // Sql table cache serializer. if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, defaultValue = false)) { conf.set( diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 6d0cdd0f8a05..47551764e632 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -174,13 +174,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf) conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString) - // FIXME: The following is a workaround. Remove once the causes are fixed. - conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString) - logWarning( - "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" + - " temporary solution to avoid OOM by Velox's global memory pools." + - " See GLUTEN-6960 for more information.") - // If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap // size. Otherwise, the off-heap size is set to the value specified by the user (if any). // Note that this means that we will IGNORE the off-heap size specified by the user if the