Support of arbitrary include files in config parser (#95)

ch_tools/common/clickhouse/config/clickhouse.py

@@ -3,10 +3,9 @@
 from ch_tools.common.clickhouse.config.storage_configuration import (
     ClickhouseStorageConfiguration,
 )
-from ch_tools.common.utils import deep_merge
 
+from ...utils import first_value
 from .path import (
-    CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH,
     CLICKHOUSE_SERVER_CONFIGD_PATH,
     CLICKHOUSE_SERVER_MAIN_CONFIG_PATH,
     CLICKHOUSE_SERVER_PREPROCESSED_CONFIG_PATH,
@@ -26,7 +25,7 @@ def __init__(self, config, preprocessed):
 
     @property
     def _config_root(self) -> dict:
-        return self._config.get("clickhouse", self._config.get("yandex", {}))
+        return first_value(self._config)
 
     @property
     def macros(self):
@@ -72,27 +71,45 @@ def load(try_preprocessed=True):
 
         # Otherwise load all server config files and perform manual merge and processing
         config = _load_config(CLICKHOUSE_SERVER_MAIN_CONFIG_PATH)
-        deep_merge(config, _load_config(CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH))
-        for file in os.listdir(CLICKHOUSE_SERVER_CONFIGD_PATH):
-            deep_merge(
-                config, _load_config(os.path.join(CLICKHOUSE_SERVER_CONFIGD_PATH, file))
-            )
+        if os.path.exists(CLICKHOUSE_SERVER_CONFIGD_PATH):
+            for file in os.listdir(CLICKHOUSE_SERVER_CONFIGD_PATH):
+                _merge_configs(
+                    config,
+                    _load_config(os.path.join(CLICKHOUSE_SERVER_CONFIGD_PATH, file)),
+                )
 
         # Process includes
-        root_key = next(iter(config))
-        root_section = config[root_key]
-        for key, config_section in root_section.copy().items():
-            if not isinstance(config_section, dict):
-                continue
+        root_section = first_value(config)
+        include_file = root_section.get("include_from")
+        if include_file:
+            include_config = first_value(_load_config(include_file))
+            _apply_config_directives(root_section, include_config)
+
+        return ClickhouseConfig(config, preprocessed=False)
 
-            include = config_section.get("@incl")
-            if not include:
-                continue
 
-            if include != key and include in root_section:
-                root_section[key] = root_section[include]
-                del root_section[include]
+def _merge_configs(main_config, additional_config):
+    for key, value in additional_config.items():
+        if key not in main_config:
+            main_config[key] = value
+            continue
 
-            del config_section["@incl"]
+        if isinstance(main_config[key], dict) and isinstance(value, dict):
+            _merge_configs(main_config[key], value)
+            continue
 
-        return ClickhouseConfig(config, preprocessed=False)
+        if value is not None:
+            main_config[key] = value
+
+
+def _apply_config_directives(config_section, include_config):
+    for key, item in config_section.items():
+        if not isinstance(item, dict):
+            continue
+
+        include = item.get("@incl")
+        if include:
+            config_section[key] = include_config[include]
+            continue
+
+        _apply_config_directives(item, include_config)

ch_tools/common/clickhouse/config/path.py

@@ -2,7 +2,6 @@
     "/var/lib/clickhouse/preprocessed_configs/config.xml"
 )
 CLICKHOUSE_SERVER_MAIN_CONFIG_PATH = "/etc/clickhouse-server/config.xml"
-CLICKHOUSE_SERVER_CLUSTER_CONFIG_PATH = "/etc/clickhouse-server/cluster.xml"
 CLICKHOUSE_SERVER_CONFIGD_PATH = "/etc/clickhouse-server/config.d"
 CLICKHOUSE_RESETUP_CONFIG_PATH = "/etc/clickhouse-server/config.d/resetup_config.xml"
 CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH = (

ch_tools/common/utils.py

@@ -1,7 +1,6 @@
 import os
 import re
 import subprocess
-from collections import OrderedDict
 from pathlib import Path
 
 
@@ -69,12 +68,16 @@ def deep_merge(dest, update):
     Like `dict.update`, but instead of updating only top-level keys, perform recursive dict merge.
     """
     for key, value in update.items():
-        if (
-            key in dest
-            and isinstance(dest[key], (dict, OrderedDict))
-            and isinstance(value, (dict, OrderedDict))
-        ):
+        if key in dest and isinstance(dest[key], dict) and isinstance(value, dict):
             deep_merge(dest[key], value)
         else:
             dest[key] = value
     return dest
+
+
+def first_key(mapping):
+    return next(iter(mapping.keys()))
+
+
+def first_value(mapping):
+    return next(iter(mapping.values()))

pyproject.toml

@@ -111,6 +111,7 @@ docker = "*"
 docker-compose = "*"
 pyhamcrest = "*"
 pytest = "*"
+pyfakefs = "*"
 
 # lint
 black = { version = "^22.0.0", python = ">=3.10" }