From 279c381d7d3211c7e36a8f7649b0a844aec8f643 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Mon, 3 Jun 2024 14:33:33 +0800 Subject: [PATCH] Fix ch --- .github/workflows/check_license.yml | 2 - .github/workflows/code_style.yml | 14 +- cpp-ch/CMakeLists.txt | 160 +++++++------- cpp-ch/local-engine/CMakeLists.txt | 208 +++++++++--------- cpp-ch/local-engine/Functions/CMakeLists.txt | 71 +++--- .../Storages/Parquet/CMakeLists.txt | 15 +- .../Storages/SubstraitSource/CMakeLists.txt | 32 ++- cpp-ch/local-engine/examples/CMakeLists.txt | 5 +- cpp-ch/local-engine/proto/CMakeLists.txt | 47 ++-- cpp-ch/local-engine/tests/CMakeLists.txt | 132 ++++++----- 10 files changed, 354 insertions(+), 332 deletions(-) diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml index f2402ed1ea663..4b2d946ed8fdb 100644 --- a/.github/workflows/check_license.yml +++ b/.github/workflows/check_license.yml @@ -14,10 +14,8 @@ # limitations under the License. name: license header check - on: pull_request - concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index bea31a99ee74d..2be2973724b7c 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -14,13 +14,12 @@ # limitations under the License. name: Code style checks - on: pull_request: paths: - '.github/workflows/code_style.yml' - 'cpp/**' - + - 'cpp-ch/**' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true @@ -29,13 +28,20 @@ jobs: CPP-format-check: name: CPP Format Check runs-on: ubuntu-latest + strategy: + matrix: + path: + - check: 'cpp' + exclude: '' + - check: 'cpp-ch' + exclude: '' steps: - uses: actions/checkout@v4 - name: Run clang-format style check for C/C++ uses: jidicula/clang-format-action@v4.11.0 with: clang-format-version: '15' - check-path: './cpp/' + check-path: ${{ matrix.path['check'] }} fallback-style: 'Google' # optional CMake-format-check: @@ -53,7 +59,7 @@ jobs: run: | git config --global --add safe.directory $GITHUB_WORKSPACE cd $GITHUB_WORKSPACE/ - fileList=$(find ./cpp -name CMakeLists.txt -o -name *.cmake) + fileList=$(find ./cpp ./cpp-ch -name CMakeLists.txt -o -name *.cmake) for file in $fileList; do /github/home/.local/bin/cmake-format --first-comment-is-literal True --in-place $file done diff --git a/cpp-ch/CMakeLists.txt b/cpp-ch/CMakeLists.txt index 0c41f494d2a1a..1ef19b3b5651a 100644 --- a/cpp-ch/CMakeLists.txt +++ b/cpp-ch/CMakeLists.txt @@ -16,7 +16,9 @@ cmake_minimum_required(VERSION 3.20) file(READ ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse.version CH_VERSION) -set(CH_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ClickHouse CACHE STRING "ClickHouse source dir") +set(CH_SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/ClickHouse + CACHE STRING "ClickHouse source dir") string(REGEX REPLACE ".*CH_ORG=([^\n]+).*" "\\1" CH_ORG "${CH_VERSION}") string(REGEX REPLACE ".*CH_BRANCH=([^\n]+).*" "\\1" CH_BRANCH "${CH_VERSION}") string(REGEX REPLACE ".*CH_COMMIT=([^\n]+).*" "\\1" CH_COMMIT "${CH_VERSION}") @@ -27,93 +29,93 @@ message("CH_COMMIT=${CH_COMMIT}") project(libch LANGUAGES C CXX ASM) file(GLOB clickhouse_files "${CH_SOURCE_DIR}/*") -if ("${CH_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}/ClickHouse") - if (NOT clickhouse_files) - execute_process(COMMAND git clone -b ${CH_BRANCH} --depth 3 https://github.com/${CH_ORG}/ClickHouse.git ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - execute_process(COMMAND git reset --hard ${CH_COMMIT} WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - execute_process(COMMAND git submodule update --init --force --depth 1 WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - else() - execute_process(COMMAND git fetch origin ${CH_BRANCH} --depth 3 WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - execute_process(COMMAND git checkout ${CH_BRANCH} WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - execute_process(COMMAND git reset --hard ${CH_COMMIT} WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - execute_process(COMMAND git submodule update --init --recursive --force --depth 1 WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) - endif() +if("${CH_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}/ClickHouse") + if(NOT clickhouse_files) + execute_process( + COMMAND + git clone -b ${CH_BRANCH} --depth 3 + https://github.com/${CH_ORG}/ClickHouse.git ${CH_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND git reset --hard ${CH_COMMIT} + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND git submodule update --init --force --depth 1 + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + else() + execute_process( + COMMAND git fetch origin ${CH_BRANCH} --depth 3 + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND git checkout ${CH_BRANCH} + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND git reset --hard ${CH_COMMIT} + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + execute_process( + COMMAND git submodule update --init --recursive --force --depth 1 + WORKING_DIRECTORY ${CH_SOURCE_DIR} COMMAND_ERROR_IS_FATAL ANY) + endif() else() - if (NOT clickhouse_files) - # Checking out *all* submodules takes > 5 min. Therefore, the smoke build ("FastTest") in CI initializes only the set of - # submodules minimally needed for a build and we cannot assume here that all submodules are populated. - message(ERROR "clickhouse ${CH_SOURCE_DIR} is missing or empty. to fix try run:") - message(STATUS " git clone --recursive --depth 1 https://github.com/Kyligence/ClickHouse.git ${CMAKE_SOURCE_DIR}") - endif() + if(NOT clickhouse_files) + # Checking out *all* submodules takes > 5 min. Therefore, the smoke build + # ("FastTest") in CI initializes only the set of submodules minimally needed + # for a build and we cannot assume here that all submodules are populated. + message(ERROR + "clickhouse ${CH_SOURCE_DIR} is missing or empty. to fix try run:") + message( + STATUS + " git clone --recursive --depth 1 https://github.com/Kyligence/ClickHouse.git ${CMAKE_SOURCE_DIR}" + ) + endif() endif() -if (EXISTS "${CH_SOURCE_DIR}/utils/extern-local-engine") - execute_process(COMMAND rm -rf ${CH_SOURCE_DIR}/utils/extern-local-engine) -endif () -execute_process(COMMAND ln -s ${CMAKE_CURRENT_SOURCE_DIR}/local-engine ${CH_SOURCE_DIR}/utils/extern-local-engine COMMAND_ERROR_IS_FATAL ANY) +if(EXISTS "${CH_SOURCE_DIR}/utils/extern-local-engine") + execute_process(COMMAND rm -rf ${CH_SOURCE_DIR}/utils/extern-local-engine) +endif() +execute_process( + COMMAND ln -s ${CMAKE_CURRENT_SOURCE_DIR}/local-engine + ${CH_SOURCE_DIR}/utils/extern-local-engine COMMAND_ERROR_IS_FATAL ANY) -# execute_process(COMMAND find ${CMAKE_CURRENT_SOURCE_DIR}/local-engine -regex '.*\.\(c\|cpp\|h\)' -exec clang-format-15 --verbose -i --style=file -i {} \;) +# execute_process(COMMAND find ${CMAKE_CURRENT_SOURCE_DIR}/local-engine -regex +# '.*\.\(c\|cpp\|h\)' -exec clang-format-15 --verbose -i --style=file -i {} \;) set(CH_BINARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build") option(ENABLE_CPP_TEST "Build CPP Unit test" OFF) -if (ENABLE_CPP_TEST) -add_custom_command( - USES_TERMINAL - COMMAND - bash -c - \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DENABLE_PROTOBUF=ON - -DENABLE_TESTS=ON - -DENABLE_JEMALLOC=ON - -DENABLE_MULTITARGET_CODE=ON - -DENABLE_EXTERN_LOCAL_ENGINE=ON - -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' - -S ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && - cmake --build ${CH_BINARY_DIR} --target ch unit_tests_local_engine\" - OUTPUT _build_ch) +if(ENABLE_CPP_TEST) + add_custom_command( + USES_TERMINAL + COMMAND + bash -c \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_PROTOBUF=ON + -DENABLE_TESTS=ON -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON + -DENABLE_EXTERN_LOCAL_ENGINE=ON -DCOMPILER_FLAGS='-fvisibility=hidden + -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B + ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target ch + unit_tests_local_engine\" + OUTPUT _build_ch) else() -add_custom_command( - USES_TERMINAL - COMMAND - bash -c - \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DENABLE_PROTOBUF=ON - -DENABLE_TESTS=OFF - -DENABLE_JEMALLOC=ON - -DENABLE_MULTITARGET_CODE=ON - -DENABLE_EXTERN_LOCAL_ENGINE=ON - -DENABLE_ODBC=OFF - -DENABLE_CAPNP=OFF - -DENABLE_ROCKSDB=OFF - -DENABLE_GRPC=OFF - -DENABLE_RUST=OFF - -DENABLE_H3=OFF - -DENABLE_AMQPCPP=OFF - -DENABLE_CASSANDRA=OFF - -DENABLE_KAFKA=OFF - -DENABLE_NATS=OFF - -DENABLE_LIBPQXX=OFF - -DENABLE_NURAFT=OFF - -DENABLE_DATASKETCHES=OFF - -DENABLE_SQLITE=OFF - -DENABLE_S2_GEOMETRY=OFF - -DENABLE_ANNOY=OFF - -DENABLE_ULID=OFF - -DENABLE_MYSQL=OFF - -DENABLE_BCRYPT=OFF - -DENABLE_LDAP=OFF - -DENABLE_MSGPACK=OFF - -DUSE_REPLXX=OFF - -DENABLE_CLICKHOUSE_ALL=OFF - -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' - -S ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && - cmake --build ${CH_BINARY_DIR} --target libch\" - OUTPUT _build_ch) + add_custom_command( + USES_TERMINAL + COMMAND + bash -c \"cmake -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_PROTOBUF=ON + -DENABLE_TESTS=OFF -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON + -DENABLE_EXTERN_LOCAL_ENGINE=ON -DENABLE_ODBC=OFF -DENABLE_CAPNP=OFF + -DENABLE_ROCKSDB=OFF -DENABLE_GRPC=OFF -DENABLE_RUST=OFF -DENABLE_H3=OFF + -DENABLE_AMQPCPP=OFF -DENABLE_CASSANDRA=OFF -DENABLE_KAFKA=OFF + -DENABLE_NATS=OFF -DENABLE_LIBPQXX=OFF -DENABLE_NURAFT=OFF + -DENABLE_DATASKETCHES=OFF -DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF + -DENABLE_ANNOY=OFF -DENABLE_ULID=OFF -DENABLE_MYSQL=OFF + -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF -DENABLE_MSGPACK=OFF + -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF + -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S + ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build + ${CH_BINARY_DIR} --target libch\" + OUTPUT _build_ch) endif() add_custom_target(build_ch ALL DEPENDS _build_ch) diff --git a/cpp-ch/local-engine/CMakeLists.txt b/cpp-ch/local-engine/CMakeLists.txt index 8c96c5f98f719..93ee4b8218af1 100644 --- a/cpp-ch/local-engine/CMakeLists.txt +++ b/cpp-ch/local-engine/CMakeLists.txt @@ -13,22 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. - -if (ENABLE_MULTITARGET_CODE) - add_definitions(-DENABLE_MULTITARGET_CODE=1) +if(ENABLE_MULTITARGET_CODE) + add_definitions(-DENABLE_MULTITARGET_CODE=1) else() - add_definitions(-DENABLE_MULTITARGET_CODE=0) + add_definitions(-DENABLE_MULTITARGET_CODE=0) endif() -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -ffunction-sections -fdata-sections") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -ffunction-sections -fdata-sections") -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -ffunction-sections -fdata-sections") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -ffunction-sections -fdata-sections") +set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections") -if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") +if(COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) + set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}") else() - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") -endif () + set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") +endif() set(THRIFT_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src") @@ -37,7 +39,7 @@ include(FindJava) include(UseJava) include(FindJNI) -#set(JNI_NATIVE_SOURCES local_engine_jni.cpp) +# set(JNI_NATIVE_SOURCES local_engine_jni.cpp) set(LOCALENGINE_SHARED_LIB _gluten_ch) add_subdirectory(proto) @@ -61,116 +63,124 @@ add_headers_and_sources(disks Disks) add_headers_and_sources(disks Disks/ObjectStorages) include_directories( - ${JNI_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR}/proto - ${THRIFT_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/thrift-cmake - ${CMAKE_BINARY_DIR}/contrib/llvm-project/llvm/include - ${CMAKE_CURRENT_SOURCE_DIR} - ${ClickHouse_SOURCE_DIR}/src - ${ClickHouse_SOURCE_DIR}/base - ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include - ${CMAKE_BINARY_DIR}/contrib/orc/c++/include - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-blobs/inc - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/core/azure-core/inc - ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-common/inc - ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/llvm/include - ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/utils/bazel/llvm-project-overlay/llvm/include + ${JNI_INCLUDE_DIRS} + ${CMAKE_CURRENT_BINARY_DIR}/proto + ${THRIFT_INCLUDE_DIR} + ${CMAKE_BINARY_DIR}/contrib/thrift-cmake + ${CMAKE_BINARY_DIR}/contrib/llvm-project/llvm/include + ${CMAKE_CURRENT_SOURCE_DIR} + ${ClickHouse_SOURCE_DIR}/src + ${ClickHouse_SOURCE_DIR}/base + ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include + ${CMAKE_BINARY_DIR}/contrib/orc/c++/include + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-blobs/inc + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/core/azure-core/inc + ${ClickHouse_SOURCE_DIR}/contrib/azure/sdk/storage/azure-storage-common/inc + ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/llvm/include + ${ClickHouse_SOURCE_DIR}/contrib/llvm-project/utils/bazel/llvm-project-overlay/llvm/include ) add_subdirectory(Storages/Parquet) add_subdirectory(Storages/SubstraitSource) add_subdirectory(Functions) -add_library(gluten_clickhouse_backend_libs - ${builder_sources} - ${join_sources} - ${parser_sources} - ${rewriter_sources} - ${storages_sources} - ${common_sources} - ${external_sources} - ${shuffle_sources} - ${operator_sources} - ${aggregate_functions_sources} - ${jni_sources} - ${disks_sources} -) - -target_link_libraries(gluten_clickhouse_backend_libs PUBLIC - substrait_source - clickhouse_aggregate_functions - clickhouse_functions - gluten_spark_functions - ch_contrib::xxHash -) +add_library( + gluten_clickhouse_backend_libs + ${builder_sources} + ${join_sources} + ${parser_sources} + ${rewriter_sources} + ${storages_sources} + ${common_sources} + ${external_sources} + ${shuffle_sources} + ${operator_sources} + ${aggregate_functions_sources} + ${jni_sources} + ${disks_sources}) + +target_link_libraries( + gluten_clickhouse_backend_libs + PUBLIC substrait_source clickhouse_aggregate_functions clickhouse_functions + gluten_spark_functions ch_contrib::xxHash) # Add udf sources files in sub directories to functions_sources -option(ENABLE_LOCAL_UDFS "Build UDFs in 'local-engine/Parser/*_udf' subdirectories" ON) -if (ENABLE_LOCAL_UDFS) - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} Parser/*_udf) - foreach(child ${children}) - add_headers_and_sources(local_udfs ${child}) - endforeach() -endif () - -file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} Parser/*_function_parser) +option(ENABLE_LOCAL_UDFS + "Build UDFs in 'local-engine/Parser/*_udf' subdirectories" ON) +if(ENABLE_LOCAL_UDFS) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + Parser/*_udf) + foreach(child ${children}) + add_headers_and_sources(local_udfs ${child}) + endforeach() +endif() + +file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + Parser/*_function_parser) foreach(child ${children}) - add_headers_and_sources(function_parsers ${child}) + add_headers_and_sources(function_parsers ${child}) endforeach() -# Notice: soures files under Parser/*_udf subdirectories must be built into target ${LOCALENGINE_SHARED_LIB} directly -# to make sure all function parsers are registered successly. -add_library(${LOCALENGINE_SHARED_LIB} SHARED - local_engine_jni.cpp - ${local_udfs_sources} - ${function_parsers_sources} - $) # why add clickhouse_malloc? check clickhouse PR-8046 +# Notice: soures files under Parser/*_udf subdirectories must be built into +# target ${LOCALENGINE_SHARED_LIB} directly to make sure all function parsers +# are registered successly. +add_library( + ${LOCALENGINE_SHARED_LIB} SHARED + local_engine_jni.cpp ${local_udfs_sources} ${function_parsers_sources} + $) # why add clickhouse_malloc? check + # clickhouse PR-8046 target_compile_options(${LOCALENGINE_SHARED_LIB} PUBLIC -fPIC - -Wno-shorten-64-to-32) - -target_link_libraries(${LOCALENGINE_SHARED_LIB} -PUBLIC - clickhouse_new_delete - clickhouse_common_config - clickhouse_common_io - clickhouse_parsers - clickhouse_storages_system - loggers - gluten_clickhouse_backend_libs - ch_contrib::protobuf -PRIVATE - substrait -) + -Wno-shorten-64-to-32) + +target_link_libraries( + ${LOCALENGINE_SHARED_LIB} + PUBLIC clickhouse_new_delete + clickhouse_common_config + clickhouse_common_io + clickhouse_parsers + clickhouse_storages_system + loggers + gluten_clickhouse_backend_libs + ch_contrib::protobuf + PRIVATE substrait) target_link_libraries(${LOCALENGINE_SHARED_LIB} PUBLIC ch_parquet) -if (ENABLE_JEMALLOC) - target_link_options(${LOCALENGINE_SHARED_LIB} PRIVATE - -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch.map -Wl,-Bsymbolic-functions) +if(ENABLE_JEMALLOC) + target_link_options( + ${LOCALENGINE_SHARED_LIB} PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch.map + -Wl,-Bsymbolic-functions) else() - target_link_options(${LOCALENGINE_SHARED_LIB} PRIVATE - -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch-hide-jemalloc.map) + target_link_options( + ${LOCALENGINE_SHARED_LIB} PRIVATE + -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libch-hide-jemalloc.map) endif() -if ("${CMAKE_BUILD_TYPE}" MATCHES "Debug") - set(LOCALENGINE_SHARED_LIB_NAME "libchd.so") -else () - set(LOCALENGINE_SHARED_LIB_NAME "libch.so") -endif () +if("${CMAKE_BUILD_TYPE}" MATCHES "Debug") + set(LOCALENGINE_SHARED_LIB_NAME "libchd.so") +else() + set(LOCALENGINE_SHARED_LIB_NAME "libch.so") +endif() add_custom_command( - OUTPUT ${LOCALENGINE_SHARED_LIB_NAME} - COMMAND ${CMAKE_COMMAND} -E rename $ ${LOCALENGINE_SHARED_LIB_NAME} - COMMENT "Renaming $ to ${LOCALENGINE_SHARED_LIB_NAME}" - DEPENDS ${LOCALENGINE_SHARED_LIB}) + OUTPUT ${LOCALENGINE_SHARED_LIB_NAME} + COMMAND ${CMAKE_COMMAND} -E rename $ + ${LOCALENGINE_SHARED_LIB_NAME} + COMMENT + "Renaming $ to ${LOCALENGINE_SHARED_LIB_NAME}" + DEPENDS ${LOCALENGINE_SHARED_LIB}) add_custom_target(libch ALL DEPENDS ${LOCALENGINE_SHARED_LIB_NAME}) add_subdirectory(tests) -if (ENABLE_EXAMPLES) - add_subdirectory(examples) -endif() \ No newline at end of file +if(ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/cpp-ch/local-engine/Functions/CMakeLists.txt b/cpp-ch/local-engine/Functions/CMakeLists.txt index 5968c86094f72..74697315597b5 100644 --- a/cpp-ch/local-engine/Functions/CMakeLists.txt +++ b/cpp-ch/local-engine/Functions/CMakeLists.txt @@ -16,51 +16,52 @@ add_headers_and_sources(gluten_spark_functions .) add_library(gluten_spark_functions_obj OBJECT ${gluten_spark_functions_sources}) -list (APPEND PRIVATE_LIBS - boost::headers_only - pcg_random - Poco::Foundation - Poco::Util - Poco::Net - Poco::JSON - ch_contrib::cctz - ch_contrib::fmt - ch_contrib::pdqsort - ch_contrib::miniselect - ch_contrib::magic_enum - ch_contrib::double_conversion - ch_contrib::dragonbox_to_chars - ch_contrib::re2 - ch_contrib::abseil_swiss_tables - ch_contrib::sparsehash - ch_contrib::metrohash - ch_contrib::murmurhash - ch_contrib::wyhash - ch_contrib::cityhash - ch_contrib::farmhash - ch_contrib::xxHash - OpenSSL::SSL -) +list( + APPEND + PRIVATE_LIBS + boost::headers_only + pcg_random + Poco::Foundation + Poco::Util + Poco::Net + Poco::JSON + ch_contrib::cctz + ch_contrib::fmt + ch_contrib::pdqsort + ch_contrib::miniselect + ch_contrib::magic_enum + ch_contrib::double_conversion + ch_contrib::dragonbox_to_chars + ch_contrib::re2 + ch_contrib::abseil_swiss_tables + ch_contrib::sparsehash + ch_contrib::metrohash + ch_contrib::murmurhash + ch_contrib::wyhash + ch_contrib::cityhash + ch_contrib::farmhash + ch_contrib::xxHash + OpenSSL::SSL) -if (TARGET ch_contrib::vectorscan) - list (APPEND PRIVATE_LIBS ch_contrib::vectorscan) +if(TARGET ch_contrib::vectorscan) + list(APPEND PRIVATE_LIBS ch_contrib::vectorscan) endif() -if (TARGET ch_contrib::rapidjson) - list (APPEND PRIVATE_LIBS ch_contrib::rapidjson) +if(TARGET ch_contrib::rapidjson) + list(APPEND PRIVATE_LIBS ch_contrib::rapidjson) endif() -if (TARGET ch_contrib::simdjson) - list (APPEND PRIVATE_LIBS ch_contrib::simdjson) +if(TARGET ch_contrib::simdjson) + list(APPEND PRIVATE_LIBS ch_contrib::simdjson) endif() -if (TARGET ch_rust::blake3) - list (APPEND PRIVATE_LIBS ch_rust::blake3) +if(TARGET ch_rust::blake3) + list(APPEND PRIVATE_LIBS ch_rust::blake3) endif() -list (APPEND OBJECT_LIBS $) +list(APPEND OBJECT_LIBS $) target_link_libraries(gluten_spark_functions_obj PRIVATE ${PRIVATE_LIBS}) add_library(gluten_spark_functions INTERFACE) -target_link_libraries(gluten_spark_functions INTERFACE ${OBJECT_LIBS}) \ No newline at end of file +target_link_libraries(gluten_spark_functions INTERFACE ${OBJECT_LIBS}) diff --git a/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt b/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt index f3d9e14f4b9c9..bfe5387108044 100644 --- a/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt +++ b/cpp-ch/local-engine/Storages/Parquet/CMakeLists.txt @@ -18,13 +18,10 @@ set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") add_headers_and_sources(Parquet .) add_library(ch_parquet ${Parquet_sources}) -target_link_libraries(ch_parquet PUBLIC - boost::headers_only - clickhouse_common_io -) +target_link_libraries(ch_parquet PUBLIC boost::headers_only + clickhouse_common_io) -target_include_directories(ch_parquet SYSTEM BEFORE PUBLIC - ${ARROW_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src -) +target_include_directories( + ch_parquet SYSTEM BEFORE + PUBLIC ${ARROW_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src) diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt index 4e43c924f6fef..228f54255cb43 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt +++ b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt @@ -15,30 +15,24 @@ set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") - macro(add_headers_and_sources_including_cc prefix common_path) - add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) - add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.cc ${common_path}/*.h) + add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) + add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c + ${common_path}/*.cc ${common_path}/*.h) endmacro() add_headers_and_sources(substrait_source .) add_headers_and_sources_including_cc(ch_parquet arrow) add_library(substrait_source ${substrait_source_sources}) -target_compile_options(substrait_source PRIVATE - -Wno-suggest-destructor-override - -Wno-inconsistent-missing-destructor-override -) +target_compile_options( + substrait_source PRIVATE -Wno-suggest-destructor-override + -Wno-inconsistent-missing-destructor-override) -target_link_libraries(substrait_source PUBLIC - boost::headers_only - ch_contrib::protobuf - clickhouse_common_io - ch_contrib::hdfs - substrait -) +target_link_libraries( + substrait_source PUBLIC boost::headers_only ch_contrib::protobuf + clickhouse_common_io ch_contrib::hdfs substrait) -target_include_directories(substrait_source SYSTEM BEFORE PUBLIC - ${ARROW_INCLUDE_DIR} - ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src - ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src -) \ No newline at end of file +target_include_directories( + substrait_source SYSTEM BEFORE + PUBLIC ${ARROW_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src + ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src) diff --git a/cpp-ch/local-engine/examples/CMakeLists.txt b/cpp-ch/local-engine/examples/CMakeLists.txt index bbeeb98d24452..03cd3bfe3f193 100644 --- a/cpp-ch/local-engine/examples/CMakeLists.txt +++ b/cpp-ch/local-engine/examples/CMakeLists.txt @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -clickhouse_add_executable (signal_demo signal_demo.cpp) -target_link_libraries(signal_demo PRIVATE gluten_clickhouse_backend_libs loggers) \ No newline at end of file +clickhouse_add_executable(signal_demo signal_demo.cpp) +target_link_libraries(signal_demo PRIVATE gluten_clickhouse_backend_libs + loggers) diff --git a/cpp-ch/local-engine/proto/CMakeLists.txt b/cpp-ch/local-engine/proto/CMakeLists.txt index 31583ff659ed1..ffb34504af526 100644 --- a/cpp-ch/local-engine/proto/CMakeLists.txt +++ b/cpp-ch/local-engine/proto/CMakeLists.txt @@ -12,37 +12,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -file(GLOB protobuf_files - substrait/*.proto - substrait/extensions/*.proto - ) +file(GLOB protobuf_files substrait/*.proto substrait/extensions/*.proto) -FOREACH(FIL ${protobuf_files}) - file(RELATIVE_PATH FIL_RELATIVE ${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/proto/ ${FIL}) - string(REGEX REPLACE "\\.proto" "" FILE_NAME ${FIL_RELATIVE}) - LIST(APPEND SUBSTRAIT_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.cc") - LIST(APPEND SUBSTRAIT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.h") -ENDFOREACH() +foreach(FIL ${protobuf_files}) + file(RELATIVE_PATH FIL_RELATIVE + ${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/proto/ ${FIL}) + string(REGEX REPLACE "\\.proto" "" FILE_NAME ${FIL_RELATIVE}) + list(APPEND SUBSTRAIT_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.cc") + list(APPEND SUBSTRAIT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.pb.h") +endforeach() # Generate Substrait headers add_custom_command( - OUTPUT ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS} - COMMAND - $ --cpp_out ${CMAKE_CURRENT_BINARY_DIR} - --proto_path ${CMAKE_CURRENT_SOURCE_DIR} - --proto_path ${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src - ${protobuf_files} - DEPENDS ${protobuf_files} - COMMENT "Running cpp protocol buffer compiler" - VERBATIM) -add_custom_target(generate_substrait ALL DEPENDS ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS}) + OUTPUT ${SUBSTRAIT_SRCS} ${SUBSTRAIT_HEADERS} + COMMAND + $ --cpp_out ${CMAKE_CURRENT_BINARY_DIR} --proto_path + ${CMAKE_CURRENT_SOURCE_DIR} --proto_path + ${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src ${protobuf_files} + DEPENDS ${protobuf_files} + COMMENT "Running cpp protocol buffer compiler" + VERBATIM) +add_custom_target(generate_substrait ALL DEPENDS ${SUBSTRAIT_SRCS} + ${SUBSTRAIT_HEADERS}) set_source_files_properties(${SUBSTRAIT_SRCS} PROPERTIES GENERATED TRUE) add_library(substrait ${SUBSTRAIT_SRCS}) add_dependencies(substrait generate_substrait) -target_compile_options(substrait PUBLIC -fPIC - -Wno-reserved-identifier - -Wno-deprecated) -target_include_directories(substrait SYSTEM BEFORE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_options(substrait PUBLIC -fPIC -Wno-reserved-identifier + -Wno-deprecated) +target_include_directories(substrait SYSTEM BEFORE + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(substrait ch_contrib::protobuf) - diff --git a/cpp-ch/local-engine/tests/CMakeLists.txt b/cpp-ch/local-engine/tests/CMakeLists.txt index 9781a332e89cf..be02bf6234d28 100644 --- a/cpp-ch/local-engine/tests/CMakeLists.txt +++ b/cpp-ch/local-engine/tests/CMakeLists.txt @@ -12,76 +12,92 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h.in ${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/testConfig.h) -if (ENABLE_TESTS) - macro(add_gtest_sources prefix common_path) - add_glob(${prefix}_sources ${common_path}/gtest*.cpp) - endmacro() +if(ENABLE_TESTS) + macro(add_gtest_sources prefix common_path) + add_glob(${prefix}_sources ${common_path}/gtest*.cpp) + endmacro() - set(USE_INTERNAL_GTEST_LIBRARY 0) - set(BENCHMARK_ENABLE_TESTING OFF) + set(USE_INTERNAL_GTEST_LIBRARY 0) + set(BENCHMARK_ENABLE_TESTING OFF) - enable_testing() - include(CTest) + enable_testing() + include(CTest) - include_directories(${GTEST_INCLUDE_DIRS}) + include_directories(${GTEST_INCLUDE_DIRS}) - set(TEST_DATA_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/tests") - set(HAVE_POSIX_REGEX 1) - set(LOCAL_ENGINE_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine") + set(TEST_DATA_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine/tests") + set(HAVE_POSIX_REGEX 1) + set(LOCAL_ENGINE_DIR "${ClickHouse_SOURCE_DIR}/utils/extern-local-engine") - add_gtest_sources(local_engine_gtest .) - if (ENABLE_LOCAL_UDFS) - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_udf/tests) - foreach(child ${children}) - add_gtest_sources(local_engine_gtest ${child}) - endforeach() - - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_udf) - foreach(child ${children}) - add_headers_and_sources(local_engine_udf ${child}) - endforeach() - endif () + add_gtest_sources(local_engine_gtest .) + if(ENABLE_LOCAL_UDFS) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_udf/tests) + foreach(child ${children}) + add_gtest_sources(local_engine_gtest ${child}) + endforeach() - file(GLOB children CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ../Parser/*_function_parser) + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_udf) foreach(child ${children}) - add_headers_and_sources(local_engine_function_parser ${child}) + add_headers_and_sources(local_engine_udf ${child}) endforeach() + endif() + file( + GLOB children CONFIGURE_DEPENDS + RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ../Parser/*_function_parser) + foreach(child ${children}) + add_headers_and_sources(local_engine_function_parser ${child}) + endforeach() - message("local engine gtest sources: ${local_engine_gtest_sources}") - message("local engine udf sources: ${local_engine_udf_sources}") - message("local engine function parser sources: ${local_engine_function_parser_sources}") + message("local engine gtest sources: ${local_engine_gtest_sources}") + message("local engine udf sources: ${local_engine_udf_sources}") + message( + "local engine function parser sources: ${local_engine_function_parser_sources}" + ) - add_executable(unit_tests_local_engine - gluten_test_util.cpp - ${local_engine_gtest_sources} - ${local_engine_udf_sources} - ${local_engine_function_parser_sources}) - target_include_directories(unit_tests_local_engine PRIVATE - ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine - ${CMAKE_CURRENT_SOURCE_DIR} - ) - # no-unreachable-code for GTEST_SKIP - target_compile_options(unit_tests_local_engine PRIVATE -Wno-unreachable-code) - target_link_libraries(unit_tests_local_engine PRIVATE gluten_clickhouse_backend_libs clickhouse_parsers loggers ch_contrib::gmock_all ch_contrib::gtest) - target_link_libraries(unit_tests_local_engine PRIVATE ch_parquet) + add_executable( + unit_tests_local_engine + gluten_test_util.cpp ${local_engine_gtest_sources} + ${local_engine_udf_sources} ${local_engine_function_parser_sources}) + target_include_directories( + unit_tests_local_engine + PRIVATE ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine + ${CMAKE_CURRENT_SOURCE_DIR}) + # no-unreachable-code for GTEST_SKIP + target_compile_options(unit_tests_local_engine PRIVATE -Wno-unreachable-code) + target_link_libraries( + unit_tests_local_engine + PRIVATE gluten_clickhouse_backend_libs clickhouse_parsers loggers + ch_contrib::gmock_all ch_contrib::gtest) + target_link_libraries(unit_tests_local_engine PRIVATE ch_parquet) endif() -if (ENABLE_BENCHMARKS) - include_directories(benchmark_local_engine SYSTEM PUBLIC - ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine - ) - add_executable(benchmark_local_engine - gluten_test_util.cpp - benchmark_local_engine.cpp - benchmark_parquet_read.cpp - benchmark_spark_row.cpp - benchmark_unix_timestamp_function.cpp - benchmark_spark_floor_function.cpp - benchmark_cast_float_function.cpp - benchmark_to_datetime_function.cpp - benchmark_spark_divide_function.cpp) - target_link_libraries(benchmark_local_engine PRIVATE gluten_clickhouse_backend_libs ch_contrib::gbenchmark_all loggers ch_parquet) +if(ENABLE_BENCHMARKS) + include_directories(benchmark_local_engine SYSTEM PUBLIC + ${ClickHouse_SOURCE_DIR}/utils/extern-local_engine) + add_executable( + benchmark_local_engine + gluten_test_util.cpp + benchmark_local_engine.cpp + benchmark_parquet_read.cpp + benchmark_spark_row.cpp + benchmark_unix_timestamp_function.cpp + benchmark_spark_floor_function.cpp + benchmark_cast_float_function.cpp + benchmark_to_datetime_function.cpp + benchmark_spark_divide_function.cpp) + target_link_libraries( + benchmark_local_engine + PRIVATE gluten_clickhouse_backend_libs ch_contrib::gbenchmark_all loggers + ch_parquet) endif()