Skip to content

Commit

Permalink
[VL] Following #6988, move a warning from core to Velox backend
Browse files Browse the repository at this point in the history
  • Loading branch information
zhztheplayer authored Aug 26, 2024
1 parent 9bb4b28 commit 2a2d2bd
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ class VeloxListenerApi extends ListenerApi with Logging {
override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = {
val conf = pc.conf()

// FIXME: The following is a workaround. Remove once the causes are fixed.
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString)
logWarning(
"Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" +
" temporary solution to avoid OOM by Velox's global memory pools." +
" See GLUTEN-6960 for more information.")

// Sql table cache serializer.
if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, defaultValue = false)) {
conf.set(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging {
val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString)

// FIXME: The following is a workaround. Remove once the causes are fixed.
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString)
logWarning(
"Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" +
" temporary solution to avoid OOM by Velox's global memory pools." +
" See GLUTEN-6960 for more information.")

// If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap
// size. Otherwise, the off-heap size is set to the value specified by the user (if any).
// Note that this means that we will IGNORE the off-heap size specified by the user if the
Expand Down

0 comments on commit 2a2d2bd

Please sign in to comment.