From 34980ec2bcb8b62940f6e67173aafcd269226f60 Mon Sep 17 00:00:00 2001 From: rui-mo Date: Tue, 24 Dec 2024 11:00:36 +0800 Subject: [PATCH] Pass timezone --- .../datasources/velox/VeloxParquetWriterInjects.scala | 5 +++++ cpp/velox/operators/writer/VeloxParquetDataSource.cc | 1 + ep/build-velox/src/get_velox.sh | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala index b6d4f6557b82..f3e286cb1379 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala @@ -39,6 +39,11 @@ class VeloxParquetWriterInjects extends VeloxFormatWriterInjects { GlutenConfig.PARQUET_BLOCK_ROWS, GlutenConfig.getConf.columnarParquetWriteBlockRows.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_ROWS, blockRows) + sparkOptions.put( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + options.getOrElse( + SQLConf.SESSION_LOCAL_TIMEZONE.key, + SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString)) options .get(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE) .foreach(sparkOptions.put(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE, _)) diff --git a/cpp/velox/operators/writer/VeloxParquetDataSource.cc b/cpp/velox/operators/writer/VeloxParquetDataSource.cc index af8e233ad082..15082b8b8281 100644 --- a/cpp/velox/operators/writer/VeloxParquetDataSource.cc +++ b/cpp/velox/operators/writer/VeloxParquetDataSource.cc @@ -97,6 +97,7 @@ void VeloxParquetDataSource::init(const std::unordered_map( maxRowGroupRows_, maxRowGroupBytes_, [&]() { return false; }); }; + writeOption.parquetWriteTimestampTimeZone = getConfigValue(sparkConfs, kSessionTimezone, std::nullopt); auto schema = gluten::fromArrowSchema(schema_); parquetWriter_ = std::make_unique(std::move(sink_), writeOption, pool_, asRowType(schema)); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index da7e1d233e19..ee42d847b7db 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,8 @@ set -exu -VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_12_23 +VELOX_REPO=https://github.com/rui-mo/velox.git +VELOX_BRANCH=fix_tz VELOX_HOME="" OS=`uname -s`