Kyligence · binmahone · Apr 1, 2022 · Apr 1, 2022 · Apr 1, 2022 · Apr 1, 2022
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -839,6 +839,34 @@ jobs:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head
+          python3 docker_server.py --release-type head --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -894,6 +894,34 @@ jobs:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head --no-push
+          python3 docker_server.py --release-type head --no-push --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,7 +14,9 @@ jobs:
     steps:
     - name: Deploy packages and assets
       run: |
-        curl '${{ secrets.PACKAGES_RELEASE_URL }}/release/${{ github.ref }}?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' -d ''
+        GITHUB_TAG="${GITHUB_REF#refs/tags/}"
+        curl --silent --data '' \
+          '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
   ############################################################################################
   ##################################### Docker images  #######################################
   ############################################################################################

diff --git a/.gitignore b/.gitignore
@@ -166,3 +166,5 @@ tests/queries/0_stateless/*.generated-expect
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+
+utils/local-engine/tests/testConfig.h
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -178,7 +178,7 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
+option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." OFF)
 if (OS_DARWIN)
     # Disable the curl, azure, senry build on MacOS
     set (ENABLE_CURL_BUILD OFF)
@@ -422,6 +422,8 @@ endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
     include(cmake/linux/default_libs.cmake)
@@ -452,7 +454,6 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
 if (OS_LINUX AND NOT ARCH_AARCH64)
     # Slightly more efficient code can be generated
     # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
@@ -510,8 +511,10 @@ macro (clickhouse_add_executable target)
     if (${type} STREQUAL EXECUTABLE)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+            if (NOT ${target} STREQUAL Git::Git)
+                # operator::new/delete for executables (MemoryTracker stuff)
+                target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            endif()
         endif()
 
         # In case of static jemalloc, because zone_register() is located in zone.c and

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54470)
 SET(VERSION_MAJOR 23)
 SET(VERSION_MINOR 1)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 688e488e930c83eefeac4f87c4cc029cc5b231e3)
-SET(VERSION_DESCRIBE v23.1.1.1-testing)
-SET(VERSION_STRING 23.1.1.1)
+SET(VERSION_PATCH 3)
+SET(VERSION_GITHASH 8dfb1700858195fa704221e360fa0798ac6ee9ed)
+SET(VERSION_DESCRIBE v23.1.3.1-stable)
+SET(VERSION_STRING 23.1.3.1)
 # end of autochange
diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -139,7 +139,8 @@
 /* #undef JEMALLOC_MUTEX_INIT_CB */
 
 /* Non-empty if the tls_model attribute is supported. */
-#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+/*#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))*/
+#define JEMALLOC_TLS_MODEL
 
 /*
  * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables

diff --git a/programs/benchmark/CMakeLists.txt b/programs/benchmark/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_BENCHMARK_SOURCES Benchmark.cpp)
+set (CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
 
 set (CLICKHOUSE_BENCHMARK_LINK
     PRIVATE

diff --git a/programs/compressor/CMakeLists.txt b/programs/compressor/CMakeLists.txt
@@ -1,6 +1,6 @@
 # Also in utils
 
-set (CLICKHOUSE_COMPRESSOR_SOURCES Compressor.cpp)
+set (CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
 
 set (CLICKHOUSE_COMPRESSOR_LINK
     PRIVATE

diff --git a/programs/extract-from-config/CMakeLists.txt b/programs/extract-from-config/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ExtractFromConfig.cpp)
+set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
 
 set (CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK
     PRIVATE

diff --git a/programs/format/CMakeLists.txt b/programs/format/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_FORMAT_SOURCES Format.cpp)
+set (CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
 
 set (CLICKHOUSE_FORMAT_LINK
     PRIVATE

diff --git a/programs/git-import/CMakeLists.txt b/programs/git-import/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_GIT_IMPORT_SOURCES git-import.cpp)
+set (CLICKHOUSE_GIT_IMPORT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/git-import.cpp)
 
 set (CLICKHOUSE_GIT_IMPORT_LINK
     PRIVATE

diff --git a/programs/install/CMakeLists.txt b/programs/install/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_INSTALL_SOURCES Install.cpp)
+set (CLICKHOUSE_INSTALL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Install.cpp)
 
 set (CLICKHOUSE_INSTALL_LINK
     PRIVATE

diff --git a/programs/keeper-converter/CMakeLists.txt b/programs/keeper-converter/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
+set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/KeeperConverter.cpp)
 
 set (CLICKHOUSE_KEEPER_CONVERTER_LINK
     PRIVATE

diff --git a/programs/local/CMakeLists.txt b/programs/local/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_LOCAL_SOURCES LocalServer.cpp)
+set (CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
 
 set (CLICKHOUSE_LOCAL_LINK
     PRIVATE

diff --git a/programs/obfuscator/CMakeLists.txt b/programs/obfuscator/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (CLICKHOUSE_OBFUSCATOR_SOURCES Obfuscator.cpp)
+set (CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
 
 set (CLICKHOUSE_OBFUSCATOR_LINK
     PRIVATE

diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt
@@ -1,8 +1,8 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake)
 
 set(CLICKHOUSE_SERVER_SOURCES
-    MetricsTransmitter.cpp
-    Server.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
 )
 
 set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $<TARGET_FILE:clickhouse_server_configs> -Wl,${NO_WHOLE_ARCHIVE}")
@@ -36,4 +36,6 @@ clickhouse_embed_binaries(
     TARGET clickhouse_server_configs
     RESOURCES config.xml users.xml embedded.xml play.html dashboard.html js/uplot.js
 )
-add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+if(NOT CLICKHOUSE_ONE_SHARED)
+    add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+endif()
diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h
@@ -149,6 +149,8 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray>
 
     void gather(ColumnGathererStream & gatherer_stream) override;
 
+    bool canBeInsideNullable() const override { return true; }
+
     ColumnPtr compress() const override;
 
     void forEachSubcolumn(ColumnCallback callback) const override

diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h
@@ -72,6 +72,7 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap>
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
     void gather(ColumnGathererStream & gatherer_stream) override;
+    bool canBeInsideNullable() const override { return true; }
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
     void compareColumn(const IColumn & rhs, size_t rhs_row_num,
                        PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,

diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h
@@ -128,6 +128,8 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable>
 
     void gather(ColumnGathererStream & gatherer_stream) override;
 
+    bool canBeInsideNullable() const override { return true; }
+
     ColumnPtr compress() const override;
 
     void forEachSubcolumn(ColumnCallback callback) const override

diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h
@@ -75,6 +75,7 @@ class ColumnTuple final : public COWHelper<IColumn, ColumnTuple>
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
     void gather(ColumnGathererStream & gatherer_stream) override;
+    bool canBeInsideNullable() const override { return true; }
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
     void compareColumn(const IColumn & rhs, size_t rhs_row_num,
                        PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,

diff --git a/src/Common/CurrentMemoryTracker.cpp b/src/Common/CurrentMemoryTracker.cpp
@@ -36,6 +36,9 @@ MemoryTracker * getMemoryTracker()
 }
 
 using DB::current_thread;
+thread_local std::function<void(Int64, bool)> CurrentMemoryTracker::before_alloc = nullptr;
+
+thread_local std::function<void(Int64)> CurrentMemoryTracker::before_free  = nullptr;
 
 void CurrentMemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
 {
@@ -52,11 +55,12 @@ void CurrentMemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded)
         if (current_thread)
         {
             Int64 will_be = current_thread->untracked_memory + size;
-
-            if (will_be > current_thread->untracked_memory_limit)
+                if (will_be > current_thread->untracked_memory_limit)
             {
-                memory_tracker->allocImpl(will_be, throw_if_memory_exceeded);
                 current_thread->untracked_memory = 0;
+                if (before_alloc)
+                    before_alloc(will_be, throw_if_memory_exceeded);
+                memory_tracker->allocImpl(will_be, throw_if_memory_exceeded);
             }
             else
             {
@@ -106,6 +110,8 @@ void CurrentMemoryTracker::free(Int64 size)
             current_thread->untracked_memory -= size;
             if (current_thread->untracked_memory < -current_thread->untracked_memory_limit)
             {
+                if (before_free)
+                    before_free(-current_thread->untracked_memory);
                 memory_tracker->free(-current_thread->untracked_memory);
                 current_thread->untracked_memory = 0;
             }

diff --git a/src/Common/CurrentMemoryTracker.h b/src/Common/CurrentMemoryTracker.h
@@ -13,7 +13,8 @@ struct CurrentMemoryTracker
     /// This function should be called after memory deallocation.
     static void free(Int64 size);
     static void check();
-
+    static thread_local std::function<void(Int64, bool)> before_alloc;
+    static thread_local std::function<void(Int64)> before_free;
 private:
     static void allocImpl(Int64 size, bool throw_if_memory_exceeded);
 };
diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h
@@ -194,6 +194,9 @@ inline bool parseIPv6(T * &src, EOFfunction eof, unsigned char * dst, int32_t fi
             if (groups <= 1 && zptr == nullptr) /// IPv4 block can't be the first
                 return clear_dst();
 
+            if (group_start) /// first octet of IPv4 should be already parsed as an IPv6 group
+                return clear_dst();
+
             ++src;
             if (eof())
                 return clear_dst();

diff --git a/src/DataTypes/DataTypeArray.h b/src/DataTypes/DataTypeArray.h
@@ -33,7 +33,7 @@ class DataTypeArray final : public IDataType
 
     bool canBeInsideNullable() const override
     {
-        return false;
+        return true;
     }
 
     MutableColumnPtr createColumn() const override;

diff --git a/src/DataTypes/DataTypeMap.h b/src/DataTypes/DataTypeMap.h
@@ -31,7 +31,7 @@ class DataTypeMap final : public IDataType
     std::string doGetName() const override;
     const char * getFamilyName() const override { return "Map"; }
 
-    bool canBeInsideNullable() const override { return false; }
+    bool canBeInsideNullable() const override { return true; }
 
     MutableColumnPtr createColumn() const override;
 

diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/DataTypeNullable.h
@@ -41,6 +41,7 @@ class DataTypeNullable final : public IDataType
     bool onlyNull() const override;
     bool canBeInsideLowCardinality() const override { return nested_data_type->canBeInsideLowCardinality(); }
     bool canBePromoted() const override { return nested_data_type->canBePromoted(); }
+    bool canBeInsideNullable() const override { return true; }
 
     const DataTypePtr & getNestedType() const { return nested_data_type; }
 private:

diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h
@@ -34,7 +34,7 @@ class DataTypeTuple final : public IDataType
     std::string doGetName() const override;
     const char * getFamilyName() const override { return "Tuple"; }
 
-    bool canBeInsideNullable() const override { return false; }
+    bool canBeInsideNullable() const override { return true; }
     bool supportsSparseSerialization() const override { return true; }
 
     MutableColumnPtr createColumn() const override;

diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
@@ -33,6 +33,7 @@ struct FormatSettings
     bool null_as_default = true;
     bool decimal_trailing_zeros = false;
     bool defaults_for_omitted_fields = true;
+    bool use_lowercase_column_name = false;
 
     bool seekable_read = true;
     UInt64 max_rows_to_read_for_schema_inference = 100;

diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
@@ -194,7 +194,15 @@ ColumnPtr IExecutableFunction::defaultImplementationForNulls(
     if (null_presence.has_nullable)
     {
         ColumnsWithTypeAndName temporary_columns = createBlockWithNestedColumns(args);
-        auto temporary_result_type = removeNullable(result_type);
+
+        DataTypePtr temporary_result_type;
+        if (resolver)
+        {
+            auto temporary_function_base = resolver->build(temporary_columns);
+            temporary_result_type = temporary_function_base->getResultType();
+        }
+        else
+            temporary_result_type = removeNullable(result_type);
 
         auto res = executeWithoutLowCardinalityColumns(temporary_columns, temporary_result_type, input_rows_count, dry_run);
         return wrapInNullable(res, args, result_type, input_rows_count);