From 9234a3cfbb5d04692073ab8125f8c4c0c5ed334a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 21:23:17 +0200 Subject: [PATCH] Merge pull request #35308 from ClickHouse/clickhouse-keeper New package clickhouse-keeper --- docker/packager/packager | 1 + packages/clickhouse-keeper-dbg.yaml | 28 ++++++++++++++++++ packages/clickhouse-keeper.yaml | 40 ++++++++++++++++++++++++++ programs/keeper/CMakeLists.txt | 6 ---- src/Compression/CompressionFactory.cpp | 17 +++++++++-- tests/ci/build_report_check.py | 15 ++++++++-- 6 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 packages/clickhouse-keeper-dbg.yaml create mode 100644 packages/clickhouse-keeper.yaml diff --git a/docker/packager/packager b/docker/packager/packager index c05e39b1d565..f82d402d613a 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -163,6 +163,7 @@ def parse_env_variables( cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") + cmake_flags.append("-DBUILD_STANDALONE_KEEPER=ON") if is_release_build(build_type, package_type, sanitizer, split_binary): cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml new file mode 100644 index 000000000000..2c70b7ad4aa4 --- /dev/null +++ b/packages/clickhouse-keeper-dbg.yaml @@ -0,0 +1,28 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-keeper-dbg" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +description: | + debugging symbols for clickhouse-keeper + This package contains the debugging symbols for clickhouse-keeper. + +contents: +- src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug + dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-keeper-dbg/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-keeper-dbg/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-keeper-dbg/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-keeper-dbg/README.md diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml new file mode 100644 index 000000000000..e99ac30f9443 --- /dev/null +++ b/packages/clickhouse-keeper.yaml @@ -0,0 +1,40 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-keeper" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +conflicts: +- clickhouse-server +depends: +- adduser +suggests: +- clickhouse-keeper-dbg + +maintainer: "ClickHouse Dev Team " +description: | + Static clickhouse-keeper binary + A stand-alone clickhouse-keeper package + + +contents: +- src: root/etc/clickhouse-keeper + dst: /etc/clickhouse-keeper + type: config +- src: root/usr/bin/clickhouse-keeper + dst: /usr/bin/clickhouse-keeper +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-keeper/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-keeper/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-keeper/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-keeper/README.md diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 9491d503fbf9..b82b13d9607a 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -71,17 +71,11 @@ if (BUILD_STANDALONE_KEEPER) ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBuffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferFromFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedWriteBuffer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDelta.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDoubleDelta.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecEncrypted.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecGorilla.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecLZ4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecMultiple.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecNone.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecT64.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecZSTD.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionFactory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/getCompressionCodecForFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/ICompressionCodec.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/LZ4_decompress_faster.cpp diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index ca5e5176d13c..abf5e38a8c33 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -165,25 +165,36 @@ void registerCodecNone(CompressionCodecFactory & factory); void registerCodecLZ4(CompressionCodecFactory & factory); void registerCodecLZ4HC(CompressionCodecFactory & factory); void registerCodecZSTD(CompressionCodecFactory & factory); +void registerCodecMultiple(CompressionCodecFactory & factory); + + +/// Keeper use only general-purpose codecs, so we don't need these special codecs +/// in standalone build +#ifndef KEEPER_STANDALONE_BUILD + void registerCodecDelta(CompressionCodecFactory & factory); void registerCodecT64(CompressionCodecFactory & factory); void registerCodecDoubleDelta(CompressionCodecFactory & factory); void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecEncrypted(CompressionCodecFactory & factory); -void registerCodecMultiple(CompressionCodecFactory & factory); + +#endif CompressionCodecFactory::CompressionCodecFactory() { - registerCodecLZ4(*this); registerCodecNone(*this); + registerCodecLZ4(*this); registerCodecZSTD(*this); registerCodecLZ4HC(*this); + registerCodecMultiple(*this); + +#ifndef KEEPER_STANDALONE_BUILD registerCodecDelta(*this); registerCodecT64(*this); registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecEncrypted(*this); - registerCodecMultiple(*this); +#endif default_codec = get("LZ4", {}); } diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 7ac3df7c10dd..ef3493d20961 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -136,6 +136,17 @@ def get_build_name_from_file_name(file_name): build_name, ) + some_builds_are_missing = len(build_reports_map) < len(reports_order) + + if some_builds_are_missing: + logging.info( + "Expected to get %s build results, got %s", + len(reports_order), + len(build_reports_map), + ) + else: + logging.info("Got exactly %s builds", len(build_reports_map)) + build_reports = [ build_reports_map[build_name] for build_name in reports_order @@ -207,10 +218,10 @@ def get_build_name_from_file_name(file_name): if build_result.status == "success": ok_builds += 1 - if ok_builds == 0: + if ok_builds == 0 or some_builds_are_missing: summary_status = "error" - description = "{}/{} builds are OK".format(ok_builds, total_builds) + description = f"{ok_builds}/{total_builds} builds are OK" print("::notice ::Report url: {}".format(url))