From 5f1a4a8491baaad47151d24423d93a79bd5d8715 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Thu, 27 Jun 2024 08:28:56 +0000 Subject: [PATCH] use internal config to avoid modify the original one --- .../org/apache/gluten/backendsapi/velox/VeloxBackend.scala | 1 + .../scala/org/apache/spark/sql/expression/UDFResolver.scala | 3 ++- cpp/velox/config/VeloxConfig.h | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 158be10f486c..c4602ccbde1b 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -65,6 +65,7 @@ object VeloxBackendSettings extends BackendSettingsApi { val GLUTEN_VELOX_UDF_LIB_PATHS = getBackendConfigPrefix() + ".udfLibraryPaths" val GLUTEN_VELOX_DRIVER_UDF_LIB_PATHS = getBackendConfigPrefix() + ".driver.udfLibraryPaths" + val GLUTEN_VELOX_INTERNAL_UDF_LIB_PATHS = getBackendConfigPrefix() + ".internal.udfLibraryPaths" val MAXIMUM_BATCH_SIZE: Int = 32768 diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index e45e8b6fa6d7..8a5687487454 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -231,8 +231,9 @@ object UDFResolver extends Logging { udfLibPaths match { case Some(paths) => + // Set resolved paths to the internal config to parse on native side. sparkConf.set( - VeloxBackendSettings.GLUTEN_VELOX_UDF_LIB_PATHS, + VeloxBackendSettings.GLUTEN_VELOX_INTERNAL_UDF_LIB_PATHS, getAllLibraries(sparkConf, isDriver, paths)) case None => } diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index f57f1293e22e..7a96f03f4985 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -99,7 +99,7 @@ const std::string kVeloxAsyncTimeoutOnTaskStopping = const int32_t kVeloxAsyncTimeoutOnTaskStoppingDefault = 30000; // 30s // udf -const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.udfLibraryPaths"; +const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.internal.udfLibraryPaths"; // backtrace allocation const std::string kBacktraceAllocation = "spark.gluten.backtrace.allocation";