From 63f92bf6deb356c36c33d88d345e4ecac7968e5e Mon Sep 17 00:00:00 2001
From: Alexander Burmak <Alex-Burmak@yandex-team.ru>
Date: Sat, 27 Jan 2024 22:15:36 +0300
Subject: [PATCH] Support of arbitrary include files in config parser

---
 ch_tools/__init__.py                          |  2 +-
 .../common/clickhouse/config/clickhouse.py    | 30 ++++++++++---------
 ch_tools/common/clickhouse/config/path.py     |  1 -
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/ch_tools/__init__.py b/ch_tools/__init__.py
index 8526f52a..8c4fbd9c 100644
--- a/ch_tools/__init__.py
+++ b/ch_tools/__init__.py
@@ -1,3 +1,3 @@
 """A set of tools for administration and diagnostics of ClickHouse DBMS."""
 
-__version__ = "1.0.0"
+__version__ = "2.556.267758743"
diff --git a/ch_tools/common/clickhouse/config/clickhouse.py b/ch_tools/common/clickhouse/config/clickhouse.py
index bc905698..14a269c7 100644
--- a/ch_tools/common/clickhouse/config/clickhouse.py
+++ b/ch_tools/common/clickhouse/config/clickhouse.py
@@ -3,7 +3,6 @@
 from ch_tools.common.utils import deep_merge
 
 from .path import (
-    CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH,
     CLICKHOUSE_SERVER_CONFIGD_PATH,
     CLICKHOUSE_SERVER_MAIN_CONFIG_PATH,
     CLICKHOUSE_SERVER_PREPROCESSED_CONFIG_PATH,
@@ -23,7 +22,7 @@ def __init__(self, config, preprocessed):
 
     @property
     def _config_root(self) -> dict:
-        return self._config.get("clickhouse", self._config.get("yandex", {}))
+        return next(iter(self._config.values()))
 
     @property
     def macros(self):
@@ -67,7 +66,6 @@ def load(try_preprocessed=True):
 
         # Otherwise load all server config files and perform manual merge and processing
         config = _load_config(CLICKHOUSE_SERVER_MAIN_CONFIG_PATH)
-        deep_merge(config, _load_config(CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH))
         for file in os.listdir(CLICKHOUSE_SERVER_CONFIGD_PATH):
             deep_merge(
                 config, _load_config(os.path.join(CLICKHOUSE_SERVER_CONFIGD_PATH, file))
@@ -76,18 +74,22 @@ def load(try_preprocessed=True):
         # Process includes
         root_key = next(iter(config))
         root_section = config[root_key]
-        for key, config_section in root_section.copy().items():
-            if not isinstance(config_section, dict):
-                continue
+        include_file = root_section.get("include_from")
+        if include_file:
+            include_config = _load_config(include_file)
+            _apply_config_directives(root_section, include_config)
 
-            include = config_section.get("@incl")
-            if not include:
-                continue
+        return ClickhouseConfig(config, preprocessed=False)
 
-            if include != key and include in root_section:
-                root_section[key] = root_section[include]
-                del root_section[include]
 
-            del config_section["@incl"]
+def _apply_config_directives(config_section, include_config):
+    for key, item in config_section.copy().items():
+        if not isinstance(item, dict):
+            continue
 
-        return ClickhouseConfig(config, preprocessed=False)
+        include = item.get("@incl")
+        if include:
+            config_section[key] = include_config[include]
+            continue
+
+        _apply_config_directives(item, include_config)
diff --git a/ch_tools/common/clickhouse/config/path.py b/ch_tools/common/clickhouse/config/path.py
index 617002b6..44fda746 100644
--- a/ch_tools/common/clickhouse/config/path.py
+++ b/ch_tools/common/clickhouse/config/path.py
@@ -2,7 +2,6 @@
     "/var/lib/clickhouse/preprocessed_configs/config.xml"
 )
 CLICKHOUSE_SERVER_MAIN_CONFIG_PATH = "/etc/clickhouse-server/config.xml"
-CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH = "/etc/clickhouse-server/cluster.xml"
 CLICKHOUSE_SERVER_CONFIGD_PATH = "/etc/clickhouse-server/config.d"
 CLICKHOUSE_RESETUP_CONFIG_PATH = "/etc/clickhouse-server/config.d/resetup_config.xml"
 CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH = (