From 254d62e7248c214591f8499ae2f99d1b669d9dfe Mon Sep 17 00:00:00 2001 From: Zhen Li <10524738+zhli1142015@users.noreply.github.com> Date: Wed, 8 May 2024 00:53:57 +0800 Subject: [PATCH] [VL] Enable split preloading by default (#5456) Enable split preloading by default. use 1 I/O thread for each task thread --- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 3c7ddf32c71f..06d72ec57182 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -1230,9 +1230,14 @@ object GlutenConfig { val COLUMNAR_VELOX_CONNECTOR_IO_THREADS = buildStaticConf("spark.gluten.sql.columnar.backend.velox.IOThreads") .internal() - .doc("The IO threads for connector split preloading") + .doc("The Size of the IO thread pool in the Connector. This thread pool is used for split" + + " preloading and DirectBufferedInput.") .intConf - .createWithDefault(0) + .createWithDefaultFunction( + () => + SQLConf.get.getConfString("spark.executor.cores", "1").toInt / SQLConf.get + .getConfString("spark.task.cpus", "1") + .toInt) val COLUMNAR_VELOX_ASYNC_TIMEOUT = buildStaticConf("spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping")