From f51c6a129d957c5d3044e8faaa3ed7dadf9693c5 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 14 Jul 2023 11:26:12 +0800 Subject: [PATCH] [GLUTEN-2306][VL] Pass spark session timezone to velox's config (#2307) --- cpp/core/config/GlutenConfig.h | 2 ++ cpp/velox/compute/WholeStageResultIterator.cc | 1 + .../src/main/scala/io/glutenproject/GlutenConfig.scala | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index b26b39e9dec2..716db74b8c0a 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -29,6 +29,8 @@ const std::string kGlutenSaveDir = "spark.gluten.saveDir"; const std::string kCaseSensitive = "spark.sql.caseSensitive"; +const std::string kSessionTimezone = "spark.sql.session.timeZone"; + const std::string kSparkOffHeapMemory = "spark.gluten.memory.offHeap.size.in.bytes"; const std::string kSparkTaskOffHeapMemory = "spark.gluten.memory.task.offHeap.size.in.bytes"; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 89edfcbfc4c4..7b3f8b10acc0 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -248,6 +248,7 @@ std::unordered_map WholeStageResultIterator::getQueryC configs[velox::core::QueryConfig::kCastToIntByTruncate] = std::to_string(true); // To align with Spark's behavior, allow decimal in casting string to int. configs[velox::core::QueryConfig::kCastIntAllowDecimal] = std::to_string(true); + configs[velox::core::QueryConfig::kSessionTimezone] = getConfigValue(kSessionTimezone, ""); // Set the max memory of partial aggregation as 3/4 of offheap size. auto maxMemory = diff --git a/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala b/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala index 112134a20614..a4be5f042d18 100644 --- a/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala +++ b/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala @@ -323,7 +323,7 @@ object GlutenConfig { // unit: SECONDS, default 1 day val GLUTEN_RESOURCE_RELATION_EXPIRED_TIME_DEFAULT: Int = 86400 - // Supported hive/pthon/scala udf names + // Supported hive/python/scala udf names val GLUTEN_SUPPORTED_HIVE_UDFS = "spark.gluten.supported.hive.udfs" val GLUTEN_SUPPORTED_PYTHON_UDFS = "spark.gluten.supported.python.udfs" val GLUTEN_SUPPORTED_SCALA_UDFS = "spark.gluten.supported.scala.udfs" @@ -359,7 +359,8 @@ object GlutenConfig { val keys = ImmutableList.of( GLUTEN_SAVE_DIR, GLUTEN_TASK_OFFHEAP_SIZE_IN_BYTES_KEY, - GLUTEN_MAX_BATCH_SIZE_KEY + GLUTEN_MAX_BATCH_SIZE_KEY, + SQLConf.SESSION_LOCAL_TIMEZONE.key ) keys.forEach( k => {