From eaa3a5701bcc449a98f3f1a51cf391a617cd06d6 Mon Sep 17 00:00:00 2001 From: Lingfeng Zhang Date: Thu, 11 May 2023 15:46:45 +0800 Subject: [PATCH] [GLUTEN-1306][VL] feat: Link static depends via vcpkg (#1384) Added an option to manage static dependencies via vcpkg. # Install build depends on system (optional) sudo -E ./dev/vcpkg/setup-build-depends.sh # Source new gcc (centos only) source /opt/rh/devtoolset-9/enable # Setup and enable vcpkg environemt source ./dev/vcpkg/env.sh # Build gluten package as original ./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --- .github/workflows/velox_be.yml | 69 ++++++++ .gitmodules | 0 cpp/CMake/FindThrift.cmake | 158 ++++++++++++++++++ cpp/CMakeLists.txt | 6 +- cpp/velox/CMakeLists.txt | 50 +++++- cpp/velox/benchmarks/CMakeLists.txt | 2 +- cpp/velox/symbols.map | 12 ++ cpp/velox/tests/CMakeLists.txt | 4 +- dev/builddeps-veloxbe.sh | 14 +- dev/package-vcpkg.sh | 10 ++ dev/vcpkg/.dockerignore | 4 + dev/vcpkg/.gitignore | 2 + dev/vcpkg/env.sh | 10 ++ dev/vcpkg/export-manifest.sh | 22 +++ dev/vcpkg/init.sh | 50 ++++++ dev/vcpkg/ports/gflags/0001-patch-dir.patch | 13 ++ dev/vcpkg/ports/gflags/fix_cmake_config.patch | 13 ++ dev/vcpkg/ports/gflags/portfile.cmake | 43 +++++ dev/vcpkg/ports/gflags/usage | 4 + .../ports/gflags/vcpkg-cmake-wrapper.cmake | 11 ++ dev/vcpkg/ports/gflags/vcpkg.json | 8 + dev/vcpkg/ports/gsasl/portfile.cmake | 25 +++ dev/vcpkg/ports/gsasl/vcpkg.json | 7 + .../ports/gtest/001-fix-UWP-death-test.patch | 13 ++ .../ports/gtest/clang-tidy-no-lint.patch | 13 ++ dev/vcpkg/ports/gtest/fix-main-lib-path.patch | 59 +++++++ dev/vcpkg/ports/gtest/portfile.cmake | 64 +++++++ dev/vcpkg/ports/gtest/usage | 8 + .../ports/gtest/vcpkg-cmake-wrapper.cmake | 9 + dev/vcpkg/ports/gtest/vcpkg.json | 17 ++ dev/vcpkg/ports/krb5/portfile.cmake | 30 ++++ dev/vcpkg/ports/krb5/vcpkg.json | 7 + dev/vcpkg/ports/libdaxctl/fix-meson.patch | 94 +++++++++++ dev/vcpkg/ports/libdaxctl/portfile.cmake | 31 ++++ dev/vcpkg/ports/libdaxctl/vcpkg.json | 7 + dev/vcpkg/ports/libdwarf/portfile.cmake | 20 +++ dev/vcpkg/ports/libdwarf/vcpkg.json | 14 ++ dev/vcpkg/ports/libhdfs3/hdfs3.patch | 51 ++++++ dev/vcpkg/ports/libhdfs3/libhdfs3Config.cmake | 41 +++++ dev/vcpkg/ports/libhdfs3/portfile.cmake | 25 +++ dev/vcpkg/ports/libhdfs3/usage | 4 + dev/vcpkg/ports/libhdfs3/vcpkg.json | 27 +++ dev/vcpkg/ports/libkmod/portfile.cmake | 19 +++ dev/vcpkg/ports/libkmod/vcpkg.json | 6 + dev/vcpkg/ports/memkind/disable-example.patch | 13 ++ dev/vcpkg/ports/memkind/portfile.cmake | 21 +++ dev/vcpkg/ports/memkind/vcpkg.json | 11 ++ dev/vcpkg/setup-build-depends.sh | 109 ++++++++++++ dev/vcpkg/toolchain.cmake | 17 ++ dev/vcpkg/triplets/x64-linux-avx.cmake | 18 ++ dev/vcpkg/vcpkg-configuration.json | 5 + dev/vcpkg/vcpkg.json | 98 +++++++++++ docs/developers/vcpkg.md | 55 ++++++ ep/build-arrow/src/build_arrow.sh | 4 +- ep/build-velox/src/build_velox.sh | 25 ++- 55 files changed, 1448 insertions(+), 24 deletions(-) create mode 100644 .gitmodules create mode 100644 cpp/CMake/FindThrift.cmake create mode 100644 cpp/velox/symbols.map create mode 100755 dev/package-vcpkg.sh create mode 100644 dev/vcpkg/.dockerignore create mode 100644 dev/vcpkg/.gitignore create mode 100755 dev/vcpkg/env.sh create mode 100755 dev/vcpkg/export-manifest.sh create mode 100755 dev/vcpkg/init.sh create mode 100644 dev/vcpkg/ports/gflags/0001-patch-dir.patch create mode 100644 dev/vcpkg/ports/gflags/fix_cmake_config.patch create mode 100644 dev/vcpkg/ports/gflags/portfile.cmake create mode 100644 dev/vcpkg/ports/gflags/usage create mode 100644 dev/vcpkg/ports/gflags/vcpkg-cmake-wrapper.cmake create mode 100644 dev/vcpkg/ports/gflags/vcpkg.json create mode 100644 dev/vcpkg/ports/gsasl/portfile.cmake create mode 100644 dev/vcpkg/ports/gsasl/vcpkg.json create mode 100644 dev/vcpkg/ports/gtest/001-fix-UWP-death-test.patch create mode 100644 dev/vcpkg/ports/gtest/clang-tidy-no-lint.patch create mode 100644 dev/vcpkg/ports/gtest/fix-main-lib-path.patch create mode 100644 dev/vcpkg/ports/gtest/portfile.cmake create mode 100644 dev/vcpkg/ports/gtest/usage create mode 100644 dev/vcpkg/ports/gtest/vcpkg-cmake-wrapper.cmake create mode 100644 dev/vcpkg/ports/gtest/vcpkg.json create mode 100644 dev/vcpkg/ports/krb5/portfile.cmake create mode 100644 dev/vcpkg/ports/krb5/vcpkg.json create mode 100644 dev/vcpkg/ports/libdaxctl/fix-meson.patch create mode 100644 dev/vcpkg/ports/libdaxctl/portfile.cmake create mode 100644 dev/vcpkg/ports/libdaxctl/vcpkg.json create mode 100644 dev/vcpkg/ports/libdwarf/portfile.cmake create mode 100644 dev/vcpkg/ports/libdwarf/vcpkg.json create mode 100644 dev/vcpkg/ports/libhdfs3/hdfs3.patch create mode 100644 dev/vcpkg/ports/libhdfs3/libhdfs3Config.cmake create mode 100644 dev/vcpkg/ports/libhdfs3/portfile.cmake create mode 100644 dev/vcpkg/ports/libhdfs3/usage create mode 100644 dev/vcpkg/ports/libhdfs3/vcpkg.json create mode 100644 dev/vcpkg/ports/libkmod/portfile.cmake create mode 100644 dev/vcpkg/ports/libkmod/vcpkg.json create mode 100644 dev/vcpkg/ports/memkind/disable-example.patch create mode 100644 dev/vcpkg/ports/memkind/portfile.cmake create mode 100644 dev/vcpkg/ports/memkind/vcpkg.json create mode 100755 dev/vcpkg/setup-build-depends.sh create mode 100644 dev/vcpkg/toolchain.cmake create mode 100644 dev/vcpkg/triplets/x64-linux-avx.cmake create mode 100644 dev/vcpkg/vcpkg-configuration.json create mode 100644 dev/vcpkg/vcpkg.json create mode 100644 docs/developers/vcpkg.md diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 5136cb142bd3..3c82109bf8b8 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -353,3 +353,72 @@ jobs: if: ${{ always() }} run: | docker stop velox-backend-centos7-test-$GITHUB_RUN_ID || true + + velox-backend-static-build-test: + runs-on: velox-self-hosted + steps: + - uses: actions/checkout@v2 + - name: Setup docker container + run: | + EXTRA_DOCKER_OPTIONS="--name velox-backend-static-build-test-$GITHUB_RUN_ID -e NUM_THREADS=30 --volume velox-backend-vcpkg-binary-cache:/var/cache/vcpkg --env VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite --detach" \ + NON_INTERACTIVE=ON \ + MOUNT_MAVEN_CACHE=OFF \ + OS_IMAGE=centos:8 \ + OS_VERSION=8 \ + tools/gluten-te/centos/cbash.sh sleep 14400 + - name: Setup maven cache + run: | + docker cp ~/.m2/repository velox-backend-static-build-test-$GITHUB_RUN_ID:/root/.m2/ + - name: Build Gluten CPP library + run: | + docker exec -i velox-backend-static-build-test-$GITHUB_RUN_ID bash -c ' + source /env.sh && \ + cd /opt/gluten && \ + sudo -E ./dev/vcpkg/setup-build-depends.sh && \ + source ./dev/vcpkg/env.sh && \ + ./dev/builddeps-veloxbe.sh --build_test=ON --build_benchmarks=ON --enable_s3=ON --enable_hdfs=ON' + - name: Build for Spark 3.2.2 + run: | + docker exec velox-backend-static-build-test-$GITHUB_RUN_ID bash -c ' + cd /opt/gluten && \ + mvn clean install -Pspark-3.2 -Pbackends-velox -DskipTests -Dspark32.version=3.2.2 && \ + cd /opt/gluten/tools/gluten-it && \ + mvn clean package -Pspark-3.2' + - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 (centos 8) + run: | + EXTRA_DOCKER_OPTIONS="--name velox-backend-static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30" \ + NON_INTERACTIVE=ON \ + MOUNT_MAVEN_CACHE=OFF \ + OS_IMAGE=centos:8 \ + OS_VERSION=8 \ + tools/gluten-te/centos/cbash.sh 'cd /opt/gluten/tools/gluten-it \ + && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 \ + && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --cpus=32 --iterations=1' + - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 (ubuntu 20.04) + run: | + EXTRA_DOCKER_OPTIONS="--name velox-backend-static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30" \ + NON_INTERACTIVE=ON \ + MOUNT_MAVEN_CACHE=OFF \ + OS_IMAGE=ubuntu:20.04 \ + tools/gluten-te/ubuntu/cbash.sh 'cd /opt/gluten/tools/gluten-it \ + && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 \ + && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --cpus=32 --iterations=1' + - name: TPC-H SF1.0 && TPC-DS SF30.0 Parquet local spark3.2 (ubuntu 22.04) + run: | + EXTRA_DOCKER_OPTIONS="--name velox-backend-static-build-test-$GITHUB_RUN_ID-tpc -e NUM_THREADS=30" \ + NON_INTERACTIVE=ON \ + MOUNT_MAVEN_CACHE=OFF \ + OS_IMAGE=ubuntu:22.04 \ + tools/gluten-te/ubuntu/cbash.sh 'cd /opt/gluten/tools/gluten-it \ + && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=h --error-on-memleak --disable-aqe --off-heap-size=20g -s=1.0 --cpus=16 --iterations=1 \ + && GLUTEN_IT_JVM_ARGS=-Xmx50G sbin/gluten-it.sh \ + --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=50g -s=30.0 --cpus=32 --iterations=1' + - name: Exit docker container + if: ${{ always() }} + run: | + docker stop velox-backend-static-build-test-$GITHUB_RUN_ID || true \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cpp/CMake/FindThrift.cmake b/cpp/CMake/FindThrift.cmake new file mode 100644 index 000000000000..07028971d9fc --- /dev/null +++ b/cpp/CMake/FindThrift.cmake @@ -0,0 +1,158 @@ +# Copyright 2012 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# - Find Thrift (a cross platform RPC lib/tool) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Thrift_ROOT - When set, this path is inspected instead of standard library +# locations as the root of the Thrift installation. +# The environment variable THRIFT_HOME overrides this variable. +# +# This module defines +# Thrift_FOUND, whether Thrift is found or not +# Thrift_COMPILER_FOUND, whether Thrift compiler is found or not +# +# thrift::thrift, a library target to use Thrift +# thrift::compiler, a executable target to use Thrift compiler + +function(EXTRACT_THRIFT_VERSION) + if(THRIFT_INCLUDE_DIR) + file(READ "${THRIFT_INCLUDE_DIR}/thrift/config.h" THRIFT_CONFIG_H_CONTENT) + string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\"" THRIFT_VERSION_DEFINITION + "${THRIFT_CONFIG_H_CONTENT}") + string(REGEX MATCH "[0-9.]+" Thrift_VERSION "${THRIFT_VERSION_DEFINITION}") + set(Thrift_VERSION + "${Thrift_VERSION}" + PARENT_SCOPE) + else() + set(Thrift_VERSION + "" + PARENT_SCOPE) + endif() +endfunction(EXTRACT_THRIFT_VERSION) + +if(MSVC_TOOLCHAIN AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX) + if(NOT ARROW_THRIFT_USE_SHARED) + if(ARROW_USE_STATIC_CRT) + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(THRIFT_MSVC_LIB_SUFFIX "mtd") + else() + set(THRIFT_MSVC_LIB_SUFFIX "mt") + endif() + else() + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(THRIFT_MSVC_LIB_SUFFIX "mdd") + else() + set(THRIFT_MSVC_LIB_SUFFIX "md") + endif() + endif() + endif() +endif() +set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}") + +if(ARROW_THRIFT_USE_SHARED) + set(THRIFT_LIB_NAMES thrift) + if(CMAKE_IMPORT_LIBRARY_SUFFIX) + list(APPEND + THRIFT_LIB_NAMES + "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + endif() + list(APPEND + THRIFT_LIB_NAMES + "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) +else() + set(THRIFT_LIB_NAMES + "${CMAKE_STATIC_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) +endif() + +if(Thrift_ROOT) + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "include") + find_program(THRIFT_COMPILER thrift + PATHS ${Thrift_ROOT} + PATH_SUFFIXES "bin") + extract_thrift_version() +else() + # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. + # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. + find_package(PkgConfig QUIET) + pkg_check_modules(THRIFT_PC thrift) + if(THRIFT_PC_FOUND) + set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") + + list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") + + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATHS ${THRIFT_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_program(THRIFT_COMPILER thrift + HINTS ${THRIFT_PC_PREFIX} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin") + set(Thrift_VERSION ${THRIFT_PC_VERSION}) + else() + find_library(THRIFT_LIB + NAMES ${THRIFT_LIB_NAMES} + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") + find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") + extract_thrift_version() + endif() +endif() + +if(THRIFT_COMPILER) + set(Thrift_COMPILER_FOUND TRUE) +else() + set(Thrift_COMPILER_FOUND FALSE) +endif() + +find_package_handle_standard_args( + Thrift + REQUIRED_VARS THRIFT_LIB THRIFT_INCLUDE_DIR + VERSION_VAR Thrift_VERSION + HANDLE_COMPONENTS) + +if(Thrift_FOUND) + if(ARROW_THRIFT_USE_SHARED) + add_library(thrift::thrift SHARED IMPORTED) + else() + add_library(thrift::thrift STATIC IMPORTED) + endif() + set_target_properties(thrift::thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + if(WIN32 AND NOT MSVC_TOOLCHAIN) + # We don't need this for Visual C++ because Thrift uses + # "#pragma comment(lib, "Ws2_32.lib")" in + # thrift/windows/config.h for Visual C++. + set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES "ws2_32") + endif() + + if(Thrift_COMPILER_FOUND) + add_executable(thrift::compiler IMPORTED) + set_target_properties(thrift::compiler PROPERTIES IMPORTED_LOCATION + "${THRIFT_COMPILER}") + endif() +endif() diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 00c30be40b65..2b0e8525b372 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -114,8 +114,8 @@ find_package(JNI REQUIRED) if(BUILD_TESTS) set(GLUTEN_GTEST_MIN_VERSION "1.13.0") - find_package(gtest ${GLUTEN_GTEST_MIN_VERSION} CONFIG) - if(NOT gtest_FOUND) + find_package(GTest ${GLUTEN_GTEST_MIN_VERSION} CONFIG) + if(NOT GTest_FOUND) include(BuildGTest) endif() include(GoogleTest) @@ -145,7 +145,7 @@ function(ADD_TEST_CASE TEST_NAME) endif() add_executable(${TEST_NAME} ${SOURCES}) - target_link_libraries(${TEST_NAME} gluten gtest gtest_main Threads::Threads glog::glog) + target_link_libraries(${TEST_NAME} gluten GTest::gtest GTest::gtest_main Threads::Threads) target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core) if(ARG_EXTRA_LINK_LIBS) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index d5dfeef2cb2c..97b6be2ef5c9 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -112,7 +112,6 @@ macro(ADD_VELOX_DEPENDENCIES) endif() add_velox_dependency(parquet::reader::thrift "${VELOX_COMPONENTS_PATH}/dwio/parquet/thrift/libvelox_dwio_parquet_thrift.a") - add_velox_dependency(thrift "${VELOX_BUILD_PATH}/third_party/arrow_ep/src/arrow_ep-build/thrift_ep-install/lib/libthrift.a") add_velox_dependency(dwio::common "${VELOX_COMPONENTS_PATH}/dwio/common/libvelox_dwio_common.a") add_velox_dependency(functions::prestosql::types "${VELOX_COMPONENTS_PATH}/functions/prestosql/types/libvelox_presto_types.a") @@ -154,9 +153,14 @@ macro(ADD_VELOX_DEPENDENCIES) endmacro() macro(find_libhdfs3) - # Find the libhdfs3 library - set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") - find_library(LIBHDFS3_LIBRARY NAMES hdfs3 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH) + find_package(libhdfs3 CONFIG) + if(libhdfs3_FOUND AND TARGET HDFS::hdfs3) + set(LIBHDFS3_LIBRARY HDFS::hdfs3) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") + find_library(LIBHDFS3_LIBRARY NAMES hdfs3 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH) + endif() + if (NOT LIBHDFS3_LIBRARY) message(FATAL_ERROR "LIBHDFS3 Library Not Found") else() @@ -165,7 +169,13 @@ macro(find_libhdfs3) endmacro() macro(find_re2) - find_library(RE2_LIBRARY NAMES re2 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH) + find_package(re2 CONFIG) + if(re2_FOUND AND TARGET re2::re2) + set(RE2_LIBRARY re2::re2) + else() + find_library(RE2_LIBRARY NAMES re2 PATHS ${SYSTEM_LIB_PATH} ${SYSTEM_LIB64_PATH} ${SYSTEM_LIB_MULTIARCH_PATH} ${SYSTEM_LOCAL_LIB_PATH} ${SYSTEM_LOCAL_LIB64_PATH} NO_DEFAULT_PATH) + endif() + if (NOT RE2_LIBRARY) message(FATAL_ERROR "RE2 Library Not Found") else() @@ -198,6 +208,11 @@ set(VELOX_SRCS ) add_library(velox SHARED ${VELOX_SRCS}) +if(ENABLE_GLUTEN_VCPKG) + # Hide symbols of static dependencies + target_link_options(velox PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map) +endif() + target_include_directories(velox PUBLIC ${CMAKE_SYSTEM_INCLUDE_PATH} ${JNI_INCLUDE_DIRS} @@ -217,7 +232,12 @@ set_target_properties(velox PROPERTIES ## `-DCMAKE_PREFIX_PATH="${folly builded directory}" to the parameters of cmake. ## It is also applicable to other dependencies. find_package(Folly REQUIRED CONFIG) -find_package(gflags REQUIRED COMPONENTS shared CONFIG) + +if(ENABLE_GLUTEN_VCPKG) + find_package(gflags REQUIRED COMPONENTS static CONFIG) +else() + find_package(gflags REQUIRED COMPONENTS shared CONFIG) +endif() target_include_directories(velox PUBLIC ${GTEST_INCLUDE_DIRS} @@ -229,6 +249,20 @@ target_link_libraries(velox PUBLIC Folly::folly) find_re2() target_link_libraries(velox PUBLIC ${RE2_LIBRARY}) +if(ENABLE_GLUTEN_VCPKG) + find_package(Thrift CONFIG) +else() + # Prefer the shared library on system. + set(ARROW_THRIFT_USE_SHARED ON) + find_package(Thrift) +endif() + +if(Thrift_FOUND) + target_link_libraries(velox PUBLIC thrift::thrift) +else() + add_velox_dependency(thrift "${VELOX_BUILD_PATH}/third_party/arrow_ep/src/arrow_ep-build/thrift_ep-install/lib/libthrift.a") +endif() + if(BUILD_TESTS) add_subdirectory(tests) endif() @@ -236,9 +270,7 @@ endif() if(ENABLE_HDFS) add_definitions(-DENABLE_HDFS) find_libhdfs3() - add_library(libhdfs3 SHARED IMPORTED) - set_target_properties(libhdfs3 PROPERTIES IMPORTED_LOCATION "${LIBHDFS3_LIBRARY}") - target_link_libraries(velox PUBLIC libhdfs3) + target_link_libraries(velox PUBLIC "${LIBHDFS3_LIBRARY}") endif() if(BUILD_BENCHMARKS) diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt index 28895838c046..66df4a269846 100644 --- a/cpp/velox/benchmarks/CMakeLists.txt +++ b/cpp/velox/benchmarks/CMakeLists.txt @@ -20,7 +20,7 @@ find_arrow_lib(${PARQUET_LIB_NAME}) function(add_velox_benchmark BM_EXEC BM_FILE) add_executable(${BM_EXEC} ${BM_FILE} BenchmarkUtils.cc) target_include_directories(${BM_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src) - target_link_libraries(${BM_EXEC} gflags_shared Arrow::parquet velox benchmark::benchmark) + target_link_libraries(${BM_EXEC} gflags Arrow::parquet velox benchmark::benchmark) endfunction() # Query benchmark diff --git a/cpp/velox/symbols.map b/cpp/velox/symbols.map new file mode 100644 index 000000000000..553a779e81fa --- /dev/null +++ b/cpp/velox/symbols.map @@ -0,0 +1,12 @@ +{ + global: + extern "C++" { + *gluten::*; + *facebook::velox::*; + }; + + Java_io_gluten*; + local: + # Hide symbols of static dependencies + *; +}; \ No newline at end of file diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt index c869a502f73a..110cf53d7bcc 100644 --- a/cpp/velox/tests/CMakeLists.txt +++ b/cpp/velox/tests/CMakeLists.txt @@ -35,9 +35,9 @@ function(add_velox_test TEST_EXEC) endif() add_executable(${TEST_EXEC} ${SOURCES}) target_include_directories(${TEST_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src) - target_link_libraries(${TEST_EXEC} gflags_shared Arrow::parquet velox gtest gtest_main glog::glog protobuf benchmark::benchmark) + target_link_libraries(${TEST_EXEC} gflags Arrow::parquet velox GTest::gtest GTest::gtest_main glog::glog benchmark::benchmark) gtest_discover_tests(${TEST_EXEC}) - endfunction() +endfunction() # velox test add_velox_test(velox_shuffle_writer_test SOURCES VeloxShuffleWriterTest.cc) diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index fe0cf4bd3a12..6d5880c82fea 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -20,6 +20,8 @@ ENABLE_S3=OFF ENABLE_HDFS=OFF ENABLE_EP_CACHE=OFF ARROW_ENABLE_CUSTOM_CODEC=OFF +ENABLE_VCPKG=OFF + for arg in "$@" do case $arg in @@ -68,6 +70,10 @@ do --enable_ep_cache=*) ENABLE_EP_CACHE=("${arg#*=}") shift # Remove argument name from processing + ;; + --enable_vcpkg=*) + ENABLE_VCPKG=("${arg#*=}") + shift # Remove argument name from processing ;; *) OTHER_ARGUMENTS+=("$1") @@ -76,6 +82,12 @@ do esac done +if [ "$ENABLE_VCPKG" = "ON" ]; then + # vcpkg will install static depends and init build environment + envs="$("$GLUTEN_DIR/dev/vcpkg/init.sh")" + eval "$envs" +fi + ##install arrow cd $GLUTEN_DIR/ep/build-arrow/src ./get_arrow.sh --enable_custom_codec=$ARROW_ENABLE_CUSTOM_CODEC @@ -97,5 +109,3 @@ cmake -DBUILD_VELOX_BACKEND=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DBUILD_TESTS=$BUILD_TESTS -DBUILD_BENCHMARKS=$BUILD_BENCHMARKS -DBUILD_JEMALLOC=$BUILD_JEMALLOC \ -DENABLE_HBM=$ENABLE_HBM -DENABLE_QAT=$ENABLE_QAT -DENABLE_IAA=$ENABLE_IAA -DENABLE_S3=$ENABLE_S3 -DENABLE_HDFS=$ENABLE_HDFS .. make -j - - diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh new file mode 100755 index 000000000000..34391b96c51d --- /dev/null +++ b/dev/package-vcpkg.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) +GLUTEN_DIR="$CURRENT_DIR/.." + +cd "$GLUTEN_DIR" +source ./dev/vcpkg/env.sh +./dev/buildbundle-veloxbe.sh --build_test=ON --build_benchmarks=ON --enable_s3=ON --enable_hdfs=ON diff --git a/dev/vcpkg/.dockerignore b/dev/vcpkg/.dockerignore new file mode 100644 index 000000000000..803c2b9affe1 --- /dev/null +++ b/dev/vcpkg/.dockerignore @@ -0,0 +1,4 @@ +/.dockerignore +/.vcpkg +/vcpkg_installed +/README.md \ No newline at end of file diff --git a/dev/vcpkg/.gitignore b/dev/vcpkg/.gitignore new file mode 100644 index 000000000000..c7c1d99a7b48 --- /dev/null +++ b/dev/vcpkg/.gitignore @@ -0,0 +1,2 @@ +/.vcpkg +/vcpkg_installed \ No newline at end of file diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh new file mode 100755 index 000000000000..51391866c66f --- /dev/null +++ b/dev/vcpkg/env.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +if [ -z "${BASH_SOURCE[0]}" ] || [ "$0" == "${BASH_SOURCE[0]}" ]; then + echo "env.sh should only be sourced in bash" >&2 + exit 1 +fi + +SCRIPT_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +init_vcpkg_env=$("${SCRIPT_ROOT}/init.sh") +eval "$init_vcpkg_env" \ No newline at end of file diff --git a/dev/vcpkg/export-manifest.sh b/dev/vcpkg/export-manifest.sh new file mode 100755 index 000000000000..7d6d36a14d91 --- /dev/null +++ b/dev/vcpkg/export-manifest.sh @@ -0,0 +1,22 @@ +#! /bin/bash + +set -ex + +SCRIPT_ROOT="$(realpath "$(dirname "$0")")" + +if [ -z "$1" ]; then + cat << EOF >&2 +Export vcpkg manifest dir. + +usage: $0 output_dir +EOF + exit 1 +fi + +OUTPUT_DIR="$1" +[ -d "$OUTPUT_DIR" ] || mkdir -p "$OUTPUT_DIR" + +rsync -a --delete \ + --exclude=/.vcpkg \ + --exclude=/vcpkg_installed \ + "$SCRIPT_ROOT/" "${OUTPUT_DIR/%\//}" \ No newline at end of file diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh new file mode 100755 index 000000000000..7316db327b6f --- /dev/null +++ b/dev/vcpkg/init.sh @@ -0,0 +1,50 @@ +#! /bin/bash + +set -e + +exec 3>&1 >&2 + +SCRIPT_ROOT="$(realpath "$(dirname "$0")")" +VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg" +VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" +VCPKG_TRIPLET=x64-linux-avx + +cd "$SCRIPT_ROOT" + +if [ ! -d "$VCPKG_ROOT" ] || [ -z "$(ls "$VCPKG_ROOT")" ]; then + git clone https://github.com/microsoft/vcpkg.git --branch 2023.04.15 "$VCPKG_ROOT" +fi +[ -f "$VCPKG" ] || "$VCPKG_ROOT/bootstrap-vcpkg.sh" -disableMetrics + +$VCPKG install --no-print-usage \ + --triplet="${VCPKG_TRIPLET}" --host-triplet="${VCPKG_TRIPLET}" + +VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} +EXPORT_TOOLS_PATH= +EXPORT_TOOLS_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/tools/protobuf:${EXPORT_TOOLS_PATH}" + +# This scripts depends on environment $CMAKE_TOOLCHAIN_FILE, which requires +# cmake >= 3.21. If system cmake < 3.25, vcpkg will download latest cmake. We +# can use vcpkg's internal cmake if we find it. +VCPKG_CMAKE_BIN_DIR=$(echo "${VCPKG_ROOT}"/downloads/tools/cmake-*/cmake-*/bin) +if [ -f "$VCPKG_CMAKE_BIN_DIR/cmake" ]; then + EXPORT_TOOLS_PATH="${VCPKG_CMAKE_BIN_DIR}:${EXPORT_TOOLS_PATH}" +fi + +EXPORT_TOOLS_PATH=${EXPORT_TOOLS_PATH/%:/} + +cat <&3 +if [ "\${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then + export VCPKG_ROOT=${VCPKG_ROOT} + export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT} + export VCPKG_TRIPLET=${VCPKG_TRIPLET} + + export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake + export PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:\${PKG_CONFIG_PATH:-} + export PATH="${EXPORT_TOOLS_PATH}:\$PATH" + + export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT} +else + echo "Gluten's vcpkg environment is enabled" >&2 +fi +EOF \ No newline at end of file diff --git a/dev/vcpkg/ports/gflags/0001-patch-dir.patch b/dev/vcpkg/ports/gflags/0001-patch-dir.patch new file mode 100644 index 000000000000..d3aaa00fbe8e --- /dev/null +++ b/dev/vcpkg/ports/gflags/0001-patch-dir.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3709073..8e919cb 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -503,7 +503,7 @@ endif () + # ---------------------------------------------------------------------------- + # installation rules + set (EXPORT_NAME ${PACKAGE_NAME}-targets) +-file (RELATIVE_PATH INSTALL_PREFIX_REL2CONFIG_DIR "${CMAKE_INSTALL_PREFIX}/${CONFIG_INSTALL_DIR}" "${CMAKE_INSTALL_PREFIX}") ++set (INSTALL_PREFIX_REL2CONFIG_DIR "../..") + configure_file (cmake/config.cmake.in "${PROJECT_BINARY_DIR}/${PACKAGE_NAME}-config-install.cmake" @ONLY) + configure_file (cmake/version.cmake.in "${PROJECT_BINARY_DIR}/${PACKAGE_NAME}-config-version.cmake" @ONLY) + diff --git a/dev/vcpkg/ports/gflags/fix_cmake_config.patch b/dev/vcpkg/ports/gflags/fix_cmake_config.patch new file mode 100644 index 000000000000..bea561ecbada --- /dev/null +++ b/dev/vcpkg/ports/gflags/fix_cmake_config.patch @@ -0,0 +1,13 @@ +diff --git a/cmake/config.cmake.in b/cmake/config.cmake.in +index a512c2a..e7f50d0 100644 +--- a/cmake/config.cmake.in ++++ b/cmake/config.cmake.in +@@ -78,7 +78,7 @@ if (NOT DEFINED @PACKAGE_PREFIX@_NOTHREADS) + else () + set (@PACKAGE_PREFIX@_NOTHREADS FALSE) + endif () +- elseif (TARGET ${@PACKAGE_PREFIX@_TARGET_PREFIX}PACKAGE_NAME@_static OR TARGET ${@PACKAGE_PREFIX@_TARGET_PREFIX}@PACKAGE_NAME@_shared) ++ elseif (TARGET ${@PACKAGE_PREFIX@_TARGET_PREFIX}@PACKAGE_NAME@_static OR TARGET ${@PACKAGE_PREFIX@_TARGET_PREFIX}@PACKAGE_NAME@_shared) + set (@PACKAGE_PREFIX@_NOTHREADS FALSE) + else () + set (@PACKAGE_PREFIX@_NOTHREADS TRUE) diff --git a/dev/vcpkg/ports/gflags/portfile.cmake b/dev/vcpkg/ports/gflags/portfile.cmake new file mode 100644 index 000000000000..f1c22ff1aa60 --- /dev/null +++ b/dev/vcpkg/ports/gflags/portfile.cmake @@ -0,0 +1,43 @@ +if(VCPKG_CMAKE_SYSTEM_NAME STREQUAL "WindowsStore") + message(FATAL_ERROR "${PORT} does not currently support UWP") +endif() + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO gflags/gflags + REF v2.2.2 + SHA512 98c4703aab24e81fe551f7831ab797fb73d0f7dfc516addb34b9ff6d0914e5fd398207889b1ae555bac039537b1d4677067dae403b64903577078d99c1bdb447 + HEAD_REF master + PATCHES + 0001-patch-dir.patch # gflags was estimating a wrong relative path between the gflags-config.cmake file and the include path; "../.." goes from share/gflags/ to the triplet root + fix_cmake_config.patch +) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH} + PREFER_NINJA + OPTIONS + -DGFLAGS_REGISTER_BUILD_DIR:BOOL=OFF + -DGFLAGS_REGISTER_INSTALL_PREFIX:BOOL=OFF + -DBUILD_gflags_nothreads_LIB:BOOL=OFF + -DGFLAGS_USE_TARGET_NAMESPACE:BOOL=ON + -DCMAKE_DEBUG_POSTFIX=d +) + +vcpkg_install_cmake() +vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/gflags) + +if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") + file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/bin ${CURRENT_PACKAGES_DIR}/bin) +endif() + +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/share) + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${SOURCE_PATH}/COPYING.txt" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) + +vcpkg_copy_pdbs() + +vcpkg_fixup_pkgconfig() diff --git a/dev/vcpkg/ports/gflags/usage b/dev/vcpkg/ports/gflags/usage new file mode 100644 index 000000000000..885eebfd0b2a --- /dev/null +++ b/dev/vcpkg/ports/gflags/usage @@ -0,0 +1,4 @@ +The package gflags provides CMake targets: + + find_package(gflags CONFIG REQUIRED) + target_link_libraries(main PRIVATE gflags::gflags) diff --git a/dev/vcpkg/ports/gflags/vcpkg-cmake-wrapper.cmake b/dev/vcpkg/ports/gflags/vcpkg-cmake-wrapper.cmake new file mode 100644 index 000000000000..a59a15102926 --- /dev/null +++ b/dev/vcpkg/ports/gflags/vcpkg-cmake-wrapper.cmake @@ -0,0 +1,11 @@ +# find_package(gflags COMPONENTS shared) with namespace is buggy +set(GFLAGS_USE_TARGET_NAMESPACE OFF) + +_find_package(${ARGS}) + +foreach(tgt gflags gflags_shared gflags_static) + if (NOT TARGET gflags::${tgt} AND TARGET ${tgt}) + add_library(gflags::${tgt} INTERFACE IMPORTED) + target_link_libraries(gflags::${tgt} INTERFACE ${tgt}) + endif() +endforeach(tgt) diff --git a/dev/vcpkg/ports/gflags/vcpkg.json b/dev/vcpkg/ports/gflags/vcpkg.json new file mode 100644 index 000000000000..3b8d22a06ff1 --- /dev/null +++ b/dev/vcpkg/ports/gflags/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "gflags", + "version-string": "2.2.2", + "port-version": 5, + "description": "A C++ library that implements commandline flags processing", + "homepage": "https://github.com/gflags/gflags", + "supports": "!uwp" +} diff --git a/dev/vcpkg/ports/gsasl/portfile.cmake b/dev/vcpkg/ports/gsasl/portfile.cmake new file mode 100644 index 000000000000..e93e35acc38a --- /dev/null +++ b/dev/vcpkg/ports/gsasl/portfile.cmake @@ -0,0 +1,25 @@ +vcpkg_download_distfile(ARCHIVE + URLS "https://ftp.gnu.org/gnu/gsasl/gsasl-2.2.0.tar.gz" + FILENAME "gsasl-2.2.0.tar.gz" + SHA512 0ae318a8616fe675e9718a3f04f33731034f9a7ba03d83ccb1a72954ded54ced35dc7c7e173fdcb6fa0f0813f8891c6cbcedf8bf70b37d00b8ec512eb9f07f5f +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" +) + +vcpkg_configure_make( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + --disable-ntlm + --without-stringprep +) +vcpkg_install_make() +vcpkg_fixup_pkgconfig() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools") + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") \ No newline at end of file diff --git a/dev/vcpkg/ports/gsasl/vcpkg.json b/dev/vcpkg/ports/gsasl/vcpkg.json new file mode 100644 index 000000000000..cdc7f18b8fe3 --- /dev/null +++ b/dev/vcpkg/ports/gsasl/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "gsasl", + "version": "2.2.0", + "dependencies": [ + "krb5" + ] +} diff --git a/dev/vcpkg/ports/gtest/001-fix-UWP-death-test.patch b/dev/vcpkg/ports/gtest/001-fix-UWP-death-test.patch new file mode 100644 index 000000000000..572b7c7a4d01 --- /dev/null +++ b/dev/vcpkg/ports/gtest/001-fix-UWP-death-test.patch @@ -0,0 +1,13 @@ +diff --git a/googletest/src/gtest.cc b/googletest/src/gtest.cc +index a64e887c..45ff24c3 100644 +--- a/googletest/src/gtest.cc ++++ b/googletest/src/gtest.cc +@@ -5409,7 +5409,7 @@ int UnitTest::Run() { + // used for the duration of the program. + impl()->set_catch_exceptions(GTEST_FLAG_GET(catch_exceptions)); + +-#if GTEST_OS_WINDOWS ++#if GTEST_OS_WINDOWS && GTEST_HAS_DEATH_TEST + // Either the user wants Google Test to catch exceptions thrown by the + // tests or this is executing in the context of death test child + // process. In either case the user does not want to see pop-up dialogs diff --git a/dev/vcpkg/ports/gtest/clang-tidy-no-lint.patch b/dev/vcpkg/ports/gtest/clang-tidy-no-lint.patch new file mode 100644 index 000000000000..588a1d637f40 --- /dev/null +++ b/dev/vcpkg/ports/gtest/clang-tidy-no-lint.patch @@ -0,0 +1,13 @@ +diff --git a/googletest/include/gtest/gtest-matchers.h b/googletest/include/gtest/gtest-matchers.h +index 2bd3dcf6..12cd60f6 100644 +--- a/googletest/include/gtest/gtest-matchers.h ++++ b/googletest/include/gtest/gtest-matchers.h +@@ -403,6 +403,7 @@ class MatcherBase : private MatcherDescriberInterface { + }; + + void Destroy() { ++ // NOLINTNEXTLINE + if (IsShared() && buffer_.shared->Unref()) { + vtable_->shared_destroy(buffer_.shared); + } + diff --git a/dev/vcpkg/ports/gtest/fix-main-lib-path.patch b/dev/vcpkg/ports/gtest/fix-main-lib-path.patch new file mode 100644 index 000000000000..c2998f3e4479 --- /dev/null +++ b/dev/vcpkg/ports/gtest/fix-main-lib-path.patch @@ -0,0 +1,59 @@ +diff --git a/googlemock/CMakeLists.txt b/googlemock/CMakeLists.txt +index e7df8ec..1ecd596 100644 +--- a/googlemock/CMakeLists.txt ++++ b/googlemock/CMakeLists.txt +@@ -120,7 +120,8 @@ endif() + ######################################################################## + # + # Install rules +-install_project(gmock gmock_main) ++install_project(gmock) ++install_project(gmock_main) + + ######################################################################## + # +diff --git a/googletest/CMakeLists.txt b/googletest/CMakeLists.txt +index abdd98b..7d1c01d 100644 +--- a/googletest/CMakeLists.txt ++++ b/googletest/CMakeLists.txt +@@ -148,7 +148,8 @@ target_link_libraries(gtest_main PUBLIC gtest) + ######################################################################## + # + # Install rules +-install_project(gtest gtest_main) ++install_project(gtest) ++install_project(gtest_main) + + ######################################################################## + # +diff --git a/googletest/cmake/internal_utils.cmake b/googletest/cmake/internal_utils.cmake +index 8d8d60a..b08ce28 100644 +--- a/googletest/cmake/internal_utils.cmake ++++ b/googletest/cmake/internal_utils.cmake +@@ -315,11 +315,15 @@ function(install_project) + install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + # Install the project targets. ++ set (LIB_INSTALL_DST ${CMAKE_INSTALL_LIBDIR}) ++ if (${ARGN} MATCHES "_main") ++ set (LIB_INSTALL_DST ${CMAKE_INSTALL_LIBDIR}/manual-link) ++ endif() + install(TARGETS ${ARGN} + EXPORT ${targets_export_name} + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" +- ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" +- LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") ++ ARCHIVE DESTINATION "${LIB_INSTALL_DST}" ++ LIBRARY DESTINATION "${LIB_INSTALL_DST}") + if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + # Install PDBs + foreach(t ${ARGN}) +@@ -328,7 +332,7 @@ function(install_project) + get_target_property(t_pdb_output_directory ${t} PDB_OUTPUT_DIRECTORY) + install(FILES + "${t_pdb_output_directory}/\${CMAKE_INSTALL_CONFIG_NAME}/$<$:${t_pdb_name_debug}>$<$>:${t_pdb_name}>.pdb" +- DESTINATION ${CMAKE_INSTALL_LIBDIR} ++ DESTINATION ${LIB_INSTALL_DST} + OPTIONAL) + endforeach() + endif() diff --git a/dev/vcpkg/ports/gtest/portfile.cmake b/dev/vcpkg/ports/gtest/portfile.cmake new file mode 100644 index 000000000000..66b65bdf8658 --- /dev/null +++ b/dev/vcpkg/ports/gtest/portfile.cmake @@ -0,0 +1,64 @@ +if (EXISTS "${CURRENT_BUILDTREES_DIR}/src/.git") + file(REMOVE_RECURSE ${CURRENT_BUILDTREES_DIR}/src) +endif() + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO google/googletest + REF v1.13.0 + SHA512 70c0cfb1b4147bdecb467ecb22ae5b5529eec0abc085763213a796b7cdbd81d1761d12b342060539b936fa54f345d33f060601544874d6213fdde79111fa813e + HEAD_REF main + PATCHES + 001-fix-UWP-death-test.patch + clang-tidy-no-lint.patch + fix-main-lib-path.patch +) + +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" GTEST_FORCE_SHARED_CRT) + +vcpkg_cmake_configure( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -DBUILD_GMOCK=ON + -Dgtest_force_shared_crt=${GTEST_FORCE_SHARED_CRT} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/GTest) + +file( + INSTALL + "${SOURCE_PATH}/googletest/src/gtest.cc" + "${SOURCE_PATH}/googletest/src/gtest_main.cc" + "${SOURCE_PATH}/googletest/src/gtest-all.cc" + "${SOURCE_PATH}/googletest/src/gtest-assertion-result.cc" + "${SOURCE_PATH}/googletest/src/gtest-death-test.cc" + "${SOURCE_PATH}/googletest/src/gtest-filepath.cc" + "${SOURCE_PATH}/googletest/src/gtest-internal-inl.h" + "${SOURCE_PATH}/googletest/src/gtest-matchers.cc" + "${SOURCE_PATH}/googletest/src/gtest-port.cc" + "${SOURCE_PATH}/googletest/src/gtest-printers.cc" + "${SOURCE_PATH}/googletest/src/gtest-test-part.cc" + "${SOURCE_PATH}/googletest/src/gtest-typed-test.cc" + DESTINATION + ${CURRENT_PACKAGES_DIR}/src +) + +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) + +vcpkg_fixup_pkgconfig() +if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/gmock_main.pc" "libdir=\${prefix}/lib" "libdir=\${prefix}/lib/manual-link") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/gtest_main.pc" "libdir=\${prefix}/lib" "libdir=\${prefix}/lib/manual-link") +endif() +if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/gmock_main.pc" "libdir=\${prefix}/lib" "libdir=\${prefix}/lib/manual-link") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/gtest_main.pc" "libdir=\${prefix}/lib" "libdir=\${prefix}/lib/manual-link") +endif() +vcpkg_copy_pdbs() + +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/dev/vcpkg/ports/gtest/usage b/dev/vcpkg/ports/gtest/usage new file mode 100644 index 000000000000..dd974b39d589 --- /dev/null +++ b/dev/vcpkg/ports/gtest/usage @@ -0,0 +1,8 @@ +The package gtest is compatible with built-in CMake targets: + + enable_testing() + + find_package(GTest CONFIG REQUIRED) + target_link_libraries(main PRIVATE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) + + add_test(AllTestsInMain main) diff --git a/dev/vcpkg/ports/gtest/vcpkg-cmake-wrapper.cmake b/dev/vcpkg/ports/gtest/vcpkg-cmake-wrapper.cmake new file mode 100644 index 000000000000..9704c9eb1f9c --- /dev/null +++ b/dev/vcpkg/ports/gtest/vcpkg-cmake-wrapper.cmake @@ -0,0 +1,9 @@ +_find_package(${ARGS}) + +# Add unnamespaced target for velox +foreach(tgt gtest gtest_main gmock gmock_main) + if (NOT TARGET ${tgt} AND TARGET GTest::${tgt}) + add_library(${tgt} INTERFACE IMPORTED) + target_link_libraries(${tgt} INTERFACE GTest::${tgt}) + endif() +endforeach(tgt) diff --git a/dev/vcpkg/ports/gtest/vcpkg.json b/dev/vcpkg/ports/gtest/vcpkg.json new file mode 100644 index 000000000000..8f9d4ce01916 --- /dev/null +++ b/dev/vcpkg/ports/gtest/vcpkg.json @@ -0,0 +1,17 @@ +{ + "name": "gtest", + "version-semver": "1.13.0", + "description": "GoogleTest and GoogleMock testing frameworks", + "homepage": "https://github.com/google/googletest", + "license": "BSD-3-Clause", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/dev/vcpkg/ports/krb5/portfile.cmake b/dev/vcpkg/ports/krb5/portfile.cmake new file mode 100644 index 000000000000..b55b58b563dc --- /dev/null +++ b/dev/vcpkg/ports/krb5/portfile.cmake @@ -0,0 +1,30 @@ +vcpkg_download_distfile(ARCHIVE + URLS "https://web.mit.edu/kerberos/dist/krb5/1.20/krb5-1.20.tar.gz" + FILENAME "krb5-1.20.tar.gz" + SHA512 9aed84a971a4d74188468870260087ec7c3a614cceb5fe32ad7da1cb8db3d66e00df801c9f900f0131ac56eb828674b8be93df474c2d13b892b70c7977388604 +) + +# for gcc >= 10 +set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} -fcommon") + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" +) + +vcpkg_configure_make( + SOURCE_PATH ${SOURCE_PATH}/src + OPTIONS --without-keyutils +) +vcpkg_install_make() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/tools") + +foreach(KRB5_SUBDIR_TO_REMOVE bin share lib/krb5 var) + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/${KRB5_SUBDIR_TO_REMOVE}") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/${KRB5_SUBDIR_TO_REMOVE}") +endforeach() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/doc/copyright.rst") +vcpkg_fixup_pkgconfig() \ No newline at end of file diff --git a/dev/vcpkg/ports/krb5/vcpkg.json b/dev/vcpkg/ports/krb5/vcpkg.json new file mode 100644 index 000000000000..86c0a6f8155e --- /dev/null +++ b/dev/vcpkg/ports/krb5/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "krb5", + "version": "1.20", + "dependencies": [ + + ] +} diff --git a/dev/vcpkg/ports/libdaxctl/fix-meson.patch b/dev/vcpkg/ports/libdaxctl/fix-meson.patch new file mode 100644 index 000000000000..e46402f07d1e --- /dev/null +++ b/dev/vcpkg/ports/libdaxctl/fix-meson.patch @@ -0,0 +1,94 @@ +diff --git a/daxctl/lib/meson.build b/daxctl/lib/meson.build +index b79c6e5..ca5f114 100644 +--- a/daxctl/lib/meson.build ++++ b/daxctl/lib/meson.build +@@ -42,3 +42,4 @@ custom_target( + ) + + install_data('daxctl.conf', install_dir : datadir / 'daxctl') ++install_headers('../libdaxctl.h', subdir : 'daxctl') +diff --git a/meson.build b/meson.build +index 990d621..a3e4e53 100644 +--- a/meson.build ++++ b/meson.build +@@ -30,10 +30,6 @@ prefixdir = get_option('prefix') + if not prefixdir.startswith('/') + error('Prefix is not absolute: "@0@"'.format(prefixdir)) + endif +-if prefixdir != rootprefixdir and rootprefixdir != '/' and not prefixdir.strip('/').startswith(rootprefixdir.strip('/') + '/') +- error('Prefix is not below root prefix (now rootprefix=@0@ prefix=@1@)'.format( +- rootprefixdir, prefixdir)) +-endif + + libdir = join_paths(prefixdir, get_option('libdir')) + rootlibdir = get_option('rootlibdir') +@@ -140,9 +136,7 @@ versiondep = declare_dependency( + ) + + kmod = dependency('libkmod') +-libudev = dependency('libudev') + uuid = dependency('uuid') +-json = dependency('json-c') + if get_option('libtracefs').enabled() + traceevent = dependency('libtraceevent') + tracefs = dependency('libtracefs') +@@ -170,25 +164,6 @@ cc = meson.get_compiler('c') + # keyutils lacks pkgconfig + keyutils = cc.find_library('keyutils', required : get_option('keyutils')) + +-# iniparser lacks pkgconfig and its header files are either at '/usr/include' or '/usr/include/iniparser' +-# Use the path provided by user via meson configure -Diniparserdir= +-# if thats not provided then try searching for 'iniparser.h' in default system include path +-# and if that not found then as a last resort try looking at '/usr/include/iniparser' +-iniparser_headers = ['iniparser.h', 'dictionary.h'] +- +-message('Looking for iniparser include headers', iniparser_headers) +- +-iniparserdir = include_directories(includedir / get_option('iniparserdir'), is_system:true) +-iniparser = cc.find_library('iniparser', required : (get_option('iniparserdir') != '') , +- has_headers :iniparser_headers ,header_include_directories : iniparserdir) +- +-if not iniparser.found() +- iniparserdir = include_directories(includedir / 'iniparser', is_system:true) +- iniparser = cc.find_library('iniparser', required : true, has_headers : iniparser_headers, +- header_include_directories : iniparserdir) +-endif +-iniparser = declare_dependency(include_directories: iniparserdir, dependencies:iniparser) +- + conf = configuration_data() + check_headers = [ + ['HAVE_DLFCN_H', 'dlfcn.h'], +@@ -268,13 +243,6 @@ foreach ident : ['secure_getenv', '__secure_getenv'] + conf.set10('HAVE_' + ident.to_upper(), cc.has_function(ident)) + endforeach + +-conf.set10('HAVE_JSON_U64', +- cc.has_function('json_object_new_uint64', +- prefix : '''#include ''', +- dependencies : json, +- ) +-) +- + ndctlconf_dir = sysconfdir / 'ndctl.conf.d' + ndctlconf = ndctlconf_dir / 'monitor.conf' + conf.set_quoted('NDCTL_CONF_FILE', ndctlconf) +@@ -317,19 +285,6 @@ ccan = static_library('ccan', + ccan_dep = declare_dependency(link_with : ccan) + + subdir('daxctl/lib') +-subdir('ndctl/lib') +-subdir('cxl/lib') +-subdir('util') +-subdir('ndctl') +-subdir('daxctl') +-subdir('cxl') +-if get_option('docs').enabled() +- subdir('Documentation/ndctl') +- subdir('Documentation/daxctl') +- subdir('Documentation/cxl') +-endif +-subdir('test') +-subdir('contrib') + + # only support spec file generation from git builds + if version_tag == '' diff --git a/dev/vcpkg/ports/libdaxctl/portfile.cmake b/dev/vcpkg/ports/libdaxctl/portfile.cmake new file mode 100644 index 000000000000..0461a0e29943 --- /dev/null +++ b/dev/vcpkg/ports/libdaxctl/portfile.cmake @@ -0,0 +1,31 @@ +vcpkg_download_distfile(ARCHIVE + URLS "https://github.com/pmem/ndctl/archive/refs/tags/v76.1.tar.gz" + FILENAME "v76.1.tar.gz" + SHA512 76d32599df029969734276f8972f3f4bf701e471117c8a48d1f96b62c87a59ac54d59104ee62d1cbbb518a06a779677ca856df32ce6218d758a8c73daa3e5b06 +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" + PATCHES fix-meson.patch +) + +vcpkg_configure_meson( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + -Dlibtracefs=disabled + -Ddocs=disabled + -Dsystemd=disabled + -Dkeyutils=disabled + OPTIONS_RELEASE + -Drootprefix=${CURRENT_PACKAGES_DIR} + OPTIONS_DEBUG + -Drootprefix=${CURRENT_PACKAGES_DIR}/debug +) + +vcpkg_install_meson() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSES/preferred/LGPL-2.1") +vcpkg_fixup_pkgconfig() \ No newline at end of file diff --git a/dev/vcpkg/ports/libdaxctl/vcpkg.json b/dev/vcpkg/ports/libdaxctl/vcpkg.json new file mode 100644 index 000000000000..21c25305290b --- /dev/null +++ b/dev/vcpkg/ports/libdaxctl/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "libdaxctl", + "version": "76.1", + "homepage": "https://github.com/pmem/ndctl", + "description": " A 'device memory' enabling project encompassing tools and libraries for CXL, NVDIMMs, DAX, memory tiering and other platform memory device topics.", + "dependencies": ["libuuid", "libkmod"] +} diff --git a/dev/vcpkg/ports/libdwarf/portfile.cmake b/dev/vcpkg/ports/libdwarf/portfile.cmake new file mode 100644 index 000000000000..5455c522c0cc --- /dev/null +++ b/dev/vcpkg/ports/libdwarf/portfile.cmake @@ -0,0 +1,20 @@ +vcpkg_download_distfile(ARCHIVE + URLS + "https://github.com/davea42/libdwarf-code/releases/download/v0.6.0/libdwarf-0.6.0.tar.xz" + "https://www.prevanders.net/libdwarf-0.6.0.tar.xz" + FILENAME "libdwarf-0.6.0.tar.xz" + SHA512 839ba5e4162630ad804d76bd2aa86f35780a178dcda110106a5ee4fb27807fdf45f12e8bbb399ff53721121d0169a73335898f94218a1853116bb106dd455950 +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" +) + +vcpkg_cmake_configure(SOURCE_PATH "${SOURCE_PATH}") + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/libdwarf) +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") +vcpkg_fixup_pkgconfig() \ No newline at end of file diff --git a/dev/vcpkg/ports/libdwarf/vcpkg.json b/dev/vcpkg/ports/libdwarf/vcpkg.json new file mode 100644 index 000000000000..9f257bb88ce2 --- /dev/null +++ b/dev/vcpkg/ports/libdwarf/vcpkg.json @@ -0,0 +1,14 @@ +{ + "name": "libdwarf", + "version": "0.6.0", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/dev/vcpkg/ports/libhdfs3/hdfs3.patch b/dev/vcpkg/ports/libhdfs3/hdfs3.patch new file mode 100644 index 000000000000..fc0717502fd9 --- /dev/null +++ b/dev/vcpkg/ports/libhdfs3/hdfs3.patch @@ -0,0 +1,51 @@ +diff --git a/depends/libhdfs3/CMake/Platform.cmake b/depends/libhdfs3/CMake/Platform.cmake +index ea00fa3f..6538e84d 100644 +--- a/depends/libhdfs3/CMake/Platform.cmake ++++ b/depends/libhdfs3/CMake/Platform.cmake +@@ -7,7 +7,7 @@ ELSE(CMAKE_SYSTEM_NAME STREQUAL "Linux") + ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + + IF(CMAKE_COMPILER_IS_GNUCXX) +- EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_COMPILER_VERSION) ++ EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpfullversion OUTPUT_VARIABLE GCC_COMPILER_VERSION) + + IF (NOT GCC_COMPILER_VERSION) + MESSAGE(FATAL_ERROR "Cannot get gcc version") +diff --git a/depends/libhdfs3/CMakeLists.txt b/depends/libhdfs3/CMakeLists.txt +index f49e68dc..ec5650ec 100644 +--- a/depends/libhdfs3/CMakeLists.txt ++++ b/depends/libhdfs3/CMakeLists.txt +@@ -22,7 +22,6 @@ FIND_PACKAGE(LibXml2 REQUIRED) + FIND_PACKAGE(Protobuf REQUIRED) + FIND_PACKAGE(KERBEROS REQUIRED) + FIND_PACKAGE(GSasl REQUIRED) +-FIND_PACKAGE(GoogleTest REQUIRED) + INCLUDE_DIRECTORIES(${GoogleTest_INCLUDE_DIR}) + LINK_LIBRARIES(${GoogleTest_LIBRARIES}) + +diff --git a/depends/libhdfs3/src/client/InputStreamImpl.cpp b/depends/libhdfs3/src/client/InputStreamImpl.cpp +index 05f27a5a..6ea9ab48 100644 +--- a/depends/libhdfs3/src/client/InputStreamImpl.cpp ++++ b/depends/libhdfs3/src/client/InputStreamImpl.cpp +@@ -734,7 +734,7 @@ void InputStreamImpl::seekInternal(int64_t pos) { + } + + try { +- if (blockReader && pos > cursor && pos < endOfCurBlock) { ++ if (blockReader && pos > cursor && pos < endOfCurBlock && pos - cursor <= 128 * 1024) { + blockReader->skip(pos - cursor); + cursor = pos; + return; +diff --git a/depends/libhdfs3/src/common/SessionConfig.cpp b/depends/libhdfs3/src/common/SessionConfig.cpp +index 632009ec..27e1ad30 100644 +--- a/depends/libhdfs3/src/common/SessionConfig.cpp ++++ b/depends/libhdfs3/src/common/SessionConfig.cpp +@@ -138,7 +138,7 @@ SessionConfig::SessionConfig(const Config & conf) { + {&rpcAuthMethod, "hadoop.security.authentication", "simple" }, + {&kerberosCachePath, "hadoop.security.kerberos.ticket.cache.path", "" }, + {&logSeverity, "dfs.client.log.severity", "INFO" }, +- {&domainSocketPath, "dfs.domain.socket.path", ""} ++ {&domainSocketPath, "dfs.domain.socket.path", "/var/lib/hadoop-hdfs/dn_socket"} + }; + + for (size_t i = 0; i < ARRAYSIZE(boolValues); ++i) { diff --git a/dev/vcpkg/ports/libhdfs3/libhdfs3Config.cmake b/dev/vcpkg/ports/libhdfs3/libhdfs3Config.cmake new file mode 100644 index 000000000000..08c8f4a90c31 --- /dev/null +++ b/dev/vcpkg/ports/libhdfs3/libhdfs3Config.cmake @@ -0,0 +1,41 @@ +include(CMakeFindDependencyMacro) +find_dependency(Boost COMPONENTS thread chrono system atomic iostreams) +find_dependency(LibXml2) +find_dependency(Protobuf) + + +FUNCTION(SET_LIBRARY_TARGET NAMESPACE LIB_NAME DEBUG_LIB_FILE_NAME RELEASE_LIB_FILE_NAME INCLUDE_DIR) + ADD_LIBRARY(${NAMESPACE}::${LIB_NAME} STATIC IMPORTED) + SET_TARGET_PROPERTIES(${NAMESPACE}::${LIB_NAME} PROPERTIES + IMPORTED_CONFIGURATIONS "RELEASE;DEBUG" + IMPORTED_LOCATION_RELEASE "${RELEASE_LIB_FILE_NAME}" + IMPORTED_LOCATION_DEBUG "${DEBUG_LIB_FILE_NAME}" + INTERFACE_INCLUDE_DIRECTORIES "${INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "protobuf::libprotobuf;LibXml2::LibXml2;${HDFS_DEPS}" + ) + SET(${NAMESPACE}_${LIB_NAME}_FOUND 1) +ENDFUNCTION() + +GET_FILENAME_COMPONENT(ROOT "${CMAKE_CURRENT_LIST_FILE}" PATH) +GET_FILENAME_COMPONENT(ROOT "${ROOT}" PATH) +GET_FILENAME_COMPONENT(ROOT "${ROOT}" PATH) + +# TODO: Use find_dependency() +SET(HDFS_DEPS + uuid + # krb5 + krb5 + k5crypto + krb5support + com_err + # gsasl + gsasl + gssapi_krb5 +) +list(TRANSFORM HDFS_DEPS PREPEND "${ROOT}/lib/lib") +list(TRANSFORM HDFS_DEPS APPEND ".a") + +# Required by krb5 +list(APPEND HDFS_DEPS "resolv") + +SET_LIBRARY_TARGET("HDFS" "hdfs3" "${ROOT}/debug/lib/libhdfs3.a" "${ROOT}/lib/libhdfs3.a" "${ROOT}/include/hdfs") \ No newline at end of file diff --git a/dev/vcpkg/ports/libhdfs3/portfile.cmake b/dev/vcpkg/ports/libhdfs3/portfile.cmake new file mode 100644 index 000000000000..e33253f519ff --- /dev/null +++ b/dev/vcpkg/ports/libhdfs3/portfile.cmake @@ -0,0 +1,25 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO apache/hawq + REF dc6282fa4fe3613bec8694f8503c039db6bc3b66 + SHA512 661e5e6bf1c1296f78fad8fa33e72f98947c62beb36bd4d5f0efe75fb190c68c5354591ac2400deeaf2a0a15f6e25623a9064f61a7ec90d7ef0c4309781e7ef2 + HEAD_REF master + PATCHES hdfs3.patch +) + +vcpkg_configure_cmake( + SOURCE_PATH ${SOURCE_PATH}/depends/libhdfs3 + PREFER_NINJA +) + +vcpkg_install_cmake() + +vcpkg_copy_pdbs() + +file(GLOB HDFS3_SHARED_LIBS ${CURRENT_PACKAGES_DIR}/debug/lib/libhdfs3.so* ${CURRENT_PACKAGES_DIR}/lib/libhdfs3.so*) +file(REMOVE ${HDFS3_SHARED_LIBS}) + +file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include ${CURRENT_PACKAGES_DIR}/debug/share) +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) +FILE(INSTALL ${CMAKE_CURRENT_LIST_DIR}/libhdfs3Config.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) +FILE(INSTALL ${CMAKE_CURRENT_LIST_DIR}/usage DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) diff --git a/dev/vcpkg/ports/libhdfs3/usage b/dev/vcpkg/ports/libhdfs3/usage new file mode 100644 index 000000000000..780d82d25683 --- /dev/null +++ b/dev/vcpkg/ports/libhdfs3/usage @@ -0,0 +1,4 @@ +The package libhdfs3 is compatible with built-in CMake targets: + + FIND_PACKAGE(libhdfs3 REQUIRED) + TARGET_LINK_LIBRARIES(main PRIVATE HDFS::hdfs3) diff --git a/dev/vcpkg/ports/libhdfs3/vcpkg.json b/dev/vcpkg/ports/libhdfs3/vcpkg.json new file mode 100644 index 000000000000..eb49df0359e4 --- /dev/null +++ b/dev/vcpkg/ports/libhdfs3/vcpkg.json @@ -0,0 +1,27 @@ +{ + "name": "libhdfs3", + "version-date": "2019-11-05", + "port-version": 3, + "description": "Native Hadoop RPC protocol and HDFS data transfer protocol implementation", + "homepage": "https://github.com/erikmuttersbach/libhdfs3", + "supports": "!windows", + "dependencies": [ + "boost-thread", + "boost-chrono", + "boost-system", + "boost-atomic", + "boost-iostreams", + "boost-function", + "boost-bind", + "boost-crc", + "boost-functional", + { + "name": "libuuid", + "platform": "!windows & !osx" + }, + "libxml2", + "protobuf", + "krb5", + "gsasl" + ] +} diff --git a/dev/vcpkg/ports/libkmod/portfile.cmake b/dev/vcpkg/ports/libkmod/portfile.cmake new file mode 100644 index 000000000000..bf45a4b58455 --- /dev/null +++ b/dev/vcpkg/ports/libkmod/portfile.cmake @@ -0,0 +1,19 @@ +set(VCPKG_LIBRARY_LINKAGE dynamic) + +vcpkg_download_distfile(ARCHIVE + URLS "https://mirrors.edge.kernel.org/pub/linux/utils/kernel/kmod/kmod-30.tar.xz" + FILENAME "kmod-30.tar.xz" + SHA512 e2cd34e600a72e44710760dfda9364b790b8352a99eafbd43e683e4a06f37e6b5c0b5d14e7c28070e30fc5fc6ceddedf7b97f3b6c2c5c2d91204fefd630b9a3e +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" +) + +vcpkg_configure_make(SOURCE_PATH ${SOURCE_PATH}) +vcpkg_install_make() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_fixup_pkgconfig() \ No newline at end of file diff --git a/dev/vcpkg/ports/libkmod/vcpkg.json b/dev/vcpkg/ports/libkmod/vcpkg.json new file mode 100644 index 000000000000..cc77056dd93c --- /dev/null +++ b/dev/vcpkg/ports/libkmod/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "libkmod", + "version": "30", + "homepage": "https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git", + "description": "Linux kernel module management tools and library" +} diff --git a/dev/vcpkg/ports/memkind/disable-example.patch b/dev/vcpkg/ports/memkind/disable-example.patch new file mode 100644 index 000000000000..ed73ea6a9c0d --- /dev/null +++ b/dev/vcpkg/ports/memkind/disable-example.patch @@ -0,0 +1,13 @@ +diff --git a/Makefile.am b/Makefile.am +index 94937c29..dd5437c3 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -735,7 +735,7 @@ endif + + include autohbw/Makefile.mk + include test/Makefile.mk +-include examples/Makefile.mk ++# include examples/Makefile.mk + include src/Makefile.mk + include tiering/Makefile.mk + include utils/memory_matrix/Makefile.mk diff --git a/dev/vcpkg/ports/memkind/portfile.cmake b/dev/vcpkg/ports/memkind/portfile.cmake new file mode 100644 index 000000000000..b126345c7cea --- /dev/null +++ b/dev/vcpkg/ports/memkind/portfile.cmake @@ -0,0 +1,21 @@ +vcpkg_download_distfile(ARCHIVE + URLS "https://github.com/memkind/memkind/archive/refs/tags/v1.14.0.tar.gz" + FILENAME "v1.14.0.tar.gz" + SHA512 313c97c28be817abc86929565063859ec475bad31871a8ebed1f6ebca6b1c5f6b8c1d1ae947f694320b4a7c9998755681ac5d13a865ec8993440fcd8b32d7b94 +) + +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE "${ARCHIVE}" + PATCHES disable-example.patch +) + +vcpkg_configure_make( + SOURCE_PATH ${SOURCE_PATH} + COPY_SOURCE +) +vcpkg_install_make() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_fixup_pkgconfig() +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") \ No newline at end of file diff --git a/dev/vcpkg/ports/memkind/vcpkg.json b/dev/vcpkg/ports/memkind/vcpkg.json new file mode 100644 index 000000000000..e62939478d41 --- /dev/null +++ b/dev/vcpkg/ports/memkind/vcpkg.json @@ -0,0 +1,11 @@ +{ + "name": "memkind", + "version": "1.14.0", + "homepage": "https://pmem.io/memkind/", + "description": "Memkind is an easy-to-use, general-purpose allocator which helps to fully utilize various kinds of memory available in the system, including DRAM, NVDIMM, and HBM", + "dependencies": [ + "libdaxctl", + "libkmod", + "numactl" + ] +} diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh new file mode 100755 index 000000000000..2e31e24c98ab --- /dev/null +++ b/dev/vcpkg/setup-build-depends.sh @@ -0,0 +1,109 @@ +#! /bin/bash + +set -e + +## Install functions begin + +install_centos_any_maven() { + if [ -z "$(which mvn)" ]; then + maven_install_dir=/opt/maven-3.6.3 + if [ -d /opt/maven-3.6.3 ]; then + echo "Failed to install maven: ${maven_install_dir} is exists" >&2 + exit 1 + fi + + cd /tmp + wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz + tar -xvf apache-maven-3.6.3-bin.tar.gz + rm apache-maven-3.6.3-bin.tar.gz + mv apache-maven-3.6.3 "${maven_install_dir}" + ln -s "${maven_install_dir}/bin/mvn" /usr/local/bin/mvn + fi +} + +install_centos_7() { + yum -y install epel-release centos-release-scl + yum -y install \ + wget curl tar zip unzip which \ + cmake3 ninja-build perl-IPC-Cmd autoconf automake libtool \ + devtoolset-9 \ + bison \ + java-1.8.0-openjdk java-1.8.0-openjdk-devel + + # git>2.7.4 + if [[ "$(git --version)" != "git version 2."* ]]; then + [ -f /etc/yum.repos.d/ius.repo ] || yum -y install https://repo.ius.io/ius-release-el7.rpm + yum -y remove git + yum -y install git236 + fi + + # flex>=2.6.0 + if [[ "$(PATH="/usr/local/bin:$PATH" flex --version 2>&1)" != "flex 2.6."* ]]; then + yum -y install gettext-devel + FLEX_URL="https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz" + mkdir -p /tmp/flex + wget -q --max-redirect 3 -O - "${FLEX_URL}" | tar -xz -C /tmp/flex --strip-components=1 + cd /tmp/flex + ./autogen.sh + ./configure + make install + cd + rm -rf /tmp/flex + fi + + install_centos_any_maven +} + +install_centos_8() { + yum -y install \ + wget curl tar zip unzip git which \ + cmake ninja-build perl-IPC-Cmd autoconf automake libtool \ + gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ \ + flex bison python3 \ + java-1.8.0-openjdk java-1.8.0-openjdk-devel + + install_centos_any_maven +} + +install_ubuntu_20.04() { + apt-get -y install \ + wget curl tar zip unzip git \ + build-essential ccache cmake ninja-build pkg-config autoconf libtool \ + flex bison \ + openjdk-8-jdk maven +} + +install_ubuntu_22.04() { install_ubuntu_20.04; } + +## Install function end + +## Usage +if [ -n "$*" ]; then + cat <&2 + exit 1 +} + +[ "$(id -u)" == 0 ] || log_fatal "Root is required" + +[ -f /etc/os-release ] || log_fatal "Failed to detect os: /etc/os-release in not exists" +eval "$(sed -En "/^(VERSION_|)ID=/s/^/OS_/p" /etc/os-release)" + +[ -n "$OS_ID" -a -n "$OS_VERSION_ID" ] || log_fatal "Failed to detect os: ID or VERSION_ID is empty" + +INSTALL_FUNC="install_${OS_ID}_${OS_VERSION_ID}" +[ "$(type -t "$INSTALL_FUNC")" == function ] || log_fatal "Unsupport OS: ${OS_ID} ${OS_VERSION_ID}" + +set -x +$INSTALL_FUNC + +echo "Success" >&2 \ No newline at end of file diff --git a/dev/vcpkg/toolchain.cmake b/dev/vcpkg/toolchain.cmake new file mode 100644 index 000000000000..13f556007067 --- /dev/null +++ b/dev/vcpkg/toolchain.cmake @@ -0,0 +1,17 @@ +set(ENABLE_GLUTEN_VCPKG ON) +set(VCPKG_MANIFEST_DIR $ENV{VCPKG_MANIFEST_DIR}) +set(VCPKG_TARGET_TRIPLET $ENV{VCPKG_TRIPLET}) +set(VCPKG_HOST_TRIPLET $ENV{VCPKG_TRIPLET}) +set(VCPKG_INSTALLED_DIR $ENV{VCPKG_MANIFEST_DIR}/vcpkg_installed) +set(VCPKG_INSTALL_OPTIONS --no-print-usage) + +# Force read CMAKE_PREFIX_PATH from env +set(CMAKE_PREFIX_PATH $ENV{CMAKE_PREFIX_PATH}) + +include($ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake) + +set(CMAKE_EXE_LINKER_FLAGS "-static-libstdc++ -static-libgcc") +set(CMAKE_SHARED_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +# Disable boost new version warning for FindBoost module +set(Boost_NO_WARN_NEW_VERSIONS ON) \ No newline at end of file diff --git a/dev/vcpkg/triplets/x64-linux-avx.cmake b/dev/vcpkg/triplets/x64-linux-avx.cmake new file mode 100644 index 000000000000..fb43d8e6d4a8 --- /dev/null +++ b/dev/vcpkg/triplets/x64-linux-avx.cmake @@ -0,0 +1,18 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") +set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++17 -mbmi2") +set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +# Fix folly static link libstdc++ +# See: https://github.com/facebook/folly/blob/b88123c2abf4b3244ed285e6db0d4bea2d24f95f/CMakeLists.txt#L192 +if(PORT MATCHES "folly") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DFOLLY_NO_EXCEPTION_TRACER=ON") +endif() + +# Avoid build both release and debug +set(VCPKG_BUILD_TYPE "release") diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json new file mode 100644 index 000000000000..2f11d5bfe4f1 --- /dev/null +++ b/dev/vcpkg/vcpkg-configuration.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-configuration.schema.json", + "overlay-ports": [ "./ports" ], + "overlay-triplets": [ "./triplets" ] +} \ No newline at end of file diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json new file mode 100644 index 000000000000..bfb38cb8868b --- /dev/null +++ b/dev/vcpkg/vcpkg.json @@ -0,0 +1,98 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933", + "dependencies": ["jemalloc"], + "default-features": ["arrow", "velox", "velox-s3", "velox-hdfs"], + "features": { + "arrow": { + "description": "Apache Arrow", + "dependencies": [ + "libevent", + "libdwarf", + "openssl", + "orc", + "protobuf", + "rapidjson", + "snappy", + "xsimd", + "zlib", + "zstd", + "thrift", + "jemalloc" + ] + }, + "velox": { + "description": "Velox backend", + "dependencies": [ + "libevent", + "openssl", + "zstd", + "lz4", + "double-conversion", + "bzip2", + "curl", + "libxml2", + "libuuid", + "lzo", + "re2", + "gflags", + "glog", + "snappy", + "libdwarf", + "fmt", + { + "name": "folly", + "features": ["zstd", "lz4", "lzma", "snappy"] + }, + "boost-atomic", + "boost-process", + "boost-context", + "boost-date-time", + "boost-filesystem", + "boost-program-options", + "boost-regex", + "boost-system", + "boost-thread", + "boost-multiprecision", + "boost-stacktrace", + "boost-uuid", + "boost-circular-buffer", + "boost-format", + "gtest", + "xsimd", + "protobuf", + "benchmark", + "jemalloc" + ] + }, + "velox-s3": { + "description": "Velox S3 Support", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "features": [ + "identity-management", + "s3", + "sts", + "cognito-identity" + ] + } + ] + }, + "velox-hdfs": { + "description": "Velox HDFS Support", + "dependencies": [ + "libhdfs3" + ] + }, + "hbm": { + "description": "HBM", + "dependencies": [ + "memkind" + ] + } + }, + "overrides": [ + { "name": "fmt", "version": "8.0.1" } + ] +} \ No newline at end of file diff --git a/docs/developers/vcpkg.md b/docs/developers/vcpkg.md new file mode 100644 index 000000000000..9a1a3f3ea136 --- /dev/null +++ b/docs/developers/vcpkg.md @@ -0,0 +1,55 @@ +--- +layout: page +title: Setup depends via vcpkg +nav_order: 9 +parent: Developer Overview +--- + +# Setup static build environment via vcpkg + +## Setup build depends + +Please install build depends on your system to compile all libraries: + +``` sh +sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh +``` + +For CentOS user, gcc 9 needs to be enabled manually before next step: + +``` sh +# CentOS 8 +source /opt/rh/gcc-toolset-9/enable + +# CentOS 7 +source /opt/rh/devtoolset-9/enable +``` + +For unsupported linux distro, you can install the following packages from package manager. + +* zip +* tar +* wget +* curl +* git >= 2.7.4 +* gcc >= 9 +* pkg-config +* autotools +* flex >= 2.6.0 +* bison +* openjdk 8 +* maven + +## Install depends and setup build environment + +Simply run: + +``` sh +source $GLUTEN_REPO/dev/vcpkg/env.sh +``` + +This script will install all static libraries into the `$GLUTEN_REPO/dev/vcpkg/vcpkg_installed/` +directory and set the `$PATH` and `$CMAKE_TOOLCHAIN_FILE`. +This make build systems to locate the binary tools and libraries. +It will take about 15~30 minutes to download and build all dependencies from source. +You can configure [binary cache](https://learn.microsoft.com/en-us/vcpkg/users/binarycaching) to accelerate the next setup. \ No newline at end of file diff --git a/ep/build-arrow/src/build_arrow.sh b/ep/build-arrow/src/build_arrow.sh index 92fc7ec9a4d1..aea341c4ed40 100755 --- a/ep/build-arrow/src/build_arrow.sh +++ b/ep/build-arrow/src/build_arrow.sh @@ -103,12 +103,12 @@ cmake -G Ninja \ -DARROW_WITH_ZSTD=ON \ -DARROW_BUILD_SHARED=ON \ -DARROW_BOOST_USE_SHARED=OFF \ + -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DARROW_JEMALLOC=ON \ -DARROW_SIMD_LEVEL=AVX2 \ -DARROW_RUNTIME_SIMD_LEVEL=MAX \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ - -Dre2_SOURCE=AUTO \ + -DARROW_DEPENDENCY_SOURCE=AUTO \ -DCMAKE_INSTALL_PREFIX=$ARROW_INSTALL_DIR \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index ac75f27ad25e..722ce21b5267 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -53,7 +53,7 @@ done function compile { TARGET_BUILD_COMMIT=$(git rev-parse --verify HEAD) - if [ $RUN_SETUP_SCRIPT == "ON" ]; then + if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then setup fi @@ -67,14 +67,29 @@ function compile { if [ $ENABLE_S3 == "ON" ]; then COMPILE_OPTION="$COMPILE_OPTION -DVELOX_ENABLE_S3=ON" fi + + ARROW_ROOT=$CURRENT_DIR/../../build-arrow/build/arrow_install + if [ -d "$ARROW_ROOT" ]; then + COMPILE_OPTION="$COMPILE_OPTION -DArrow_DIR=${ARROW_ROOT} -DParquet_DIR=${ARROW_ROOT}" + fi + COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" COMPILE_TYPE=$(if [[ "$BUILD_TYPE" == "debug" ]] || [[ "$BUILD_TYPE" == "Debug" ]]; then echo 'debug'; else echo 'release'; fi) echo "COMPILE_OPTION: "$COMPILE_OPTION make $COMPILE_TYPE EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" - echo "INSTALL xsimd gtest." - cd _build/$COMPILE_TYPE/_deps - sudo cmake --install xsimd-build/ - sudo cmake --install gtest-build/ + + # Install deps to system as needed + if [ -d "_build/$COMPILE_TYPE/_deps" ]; then + cd _build/$COMPILE_TYPE/_deps + if [ -d xsimd-build ]; then + echo "INSTALL xsimd." + sudo cmake --install xsimd-build/ + fi + if [ -d gtest-build ]; then + echo "INSTALL gtest." + sudo cmake --install gtest-build/ + fi + fi } function check_commit {