From 8391156b92cb2905c15dd57c78f8169541bb026a Mon Sep 17 00:00:00 2001 From: Gluten Performance Bot <137994563+GlutenPerfBot@users.noreply.github.com> Date: Sat, 12 Oct 2024 21:04:29 +0800 Subject: [PATCH] [GLUTEN-6887][VL] Daily Update Velox Version (2024_10_12) (#7487) * [GLUTEN-6887][VL] Daily Update Velox Version (2024_10_12) Upstream Velox's New Commits: 5bedca0f3 by Kevin Wilfong, Add tests constructing maps with duplicate custom TimestampWithTimezones (11228) c434ed8c6 by Kevin Wilfong, Add support for custom comparison in Presto's contains UDF (11227) 903ae35d3 by Kevin Wilfong, Add support for custom comparison in Presto's array_position UDF (11226) 0fba52159 by Kevin Wilfong, Add tests for array_union and array_remove with TimestampWithTimezone (11222) f4ca8a60c by duanmeng, Ignore tracing of auxiliary operator (11220) 74a0db903 by Deepak Majeti, Remove connector factory registrations (8871) daeff5943 by Jialiang Tan, Do not pre-allocate memory when init vector stream (11211) --------- Signed-off-by: glutenperfbot Signed-off-by: Yuan Zhou Co-authored-by: glutenperfbot Co-authored-by: Yuan Zhou --- .github/workflows/check_license.yml | 2 +- cpp/velox/CMakeLists.txt | 5 +++-- cpp/velox/compute/VeloxBackend.cc | 26 ++++++++++++++++++++++++++ ep/build-velox/src/get_velox.sh | 2 +- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml index 338397dbd6cb..60aa73ec93f5 100644 --- a/.github/workflows/check_license.yml +++ b/.github/workflows/check_license.yml @@ -29,6 +29,6 @@ jobs: - name: Check License Header run: | git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}} - pip install regex + pip install regex --break-system-packages cd $GITHUB_WORKSPACE/ ./.github/workflows/util/check.sh ${{github.event.pull_request.base.sha}} diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 39b3f46b06e6..f9b11a590311 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -230,8 +230,9 @@ if(ENABLE_GLUTEN_VCPKG) endif() target_include_directories( - velox PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${PROTOBUF_INCLUDE} ${VELOX_HOME}) + velox + PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS} ${VELOX_BUILD_PATH} + ${CMAKE_CURRENT_SOURCE_DIR} ${PROTOBUF_INCLUDE} ${VELOX_HOME}) if(BUILD_TESTS) target_include_directories(velox PUBLIC ${VELOX_BUILD_PATH}) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index 5110590d0277..609ae6fce31c 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -39,6 +39,13 @@ #include "velox/common/file/FileSystems.h" #include "velox/connectors/hive/HiveConnector.h" #include "velox/connectors/hive/HiveDataSource.h" +#include "velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" // @manual +#include "velox/connectors/hive/storage_adapters/gcs/RegisterGCSFileSystem.h" // @manual +#include "velox/connectors/hive/storage_adapters/hdfs/RegisterHdfsFileSystem.h" // @manual +#include "velox/connectors/hive/storage_adapters/s3fs/RegisterS3FileSystem.h" // @manual +#include "velox/dwio/orc/reader/OrcReader.h" +#include "velox/dwio/parquet/RegisterParquetReader.h" +#include "velox/dwio/parquet/RegisterParquetWriter.h" #include "velox/serializers/PrestoSerializer.h" DECLARE_bool(velox_exception_user_stacktrace_enabled); @@ -110,10 +117,29 @@ void VeloxBackend::init(const std::unordered_map& conf // Setup and register. velox::filesystems::registerLocalFileSystem(); + +#ifdef ENABLE_HDFS + velox::filesystems::registerHdfsFileSystem(); +#endif +#ifdef ENABLE_S3 + velox::filesystems::registerS3FileSystem(); +#endif +#ifdef ENABLE_GCS + velox::filesystems::registerGCSFileSystem(); +#endif +#ifdef ENABLE_ABFS + velox::filesystems::abfs::registerAbfsFileSystem(); +#endif + initJolFilesystem(); initCache(); initConnector(); + velox::dwio::common::registerFileSinks(); + velox::parquet::registerParquetReaderFactory(); + velox::parquet::registerParquetWriterFactory(); + velox::orc::registerOrcReaderFactory(); + // Register Velox functions registerAllFunctions(); if (!facebook::velox::isRegisteredVectorSerde()) { diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 66cd43ebe4bd..8c3fe4bf817f 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_10_11 +VELOX_BRANCH=2024_10_12 VELOX_HOME="" OS=`uname -s`