Config file support

ch_tools/chadmin/chadmin_cli.py

@@ -4,12 +4,14 @@
 import warnings
 from typing import Any, List
 
+import cloup
+
+from ch_tools.common.config import load_config
+
 warnings.filterwarnings(action="ignore", message="Python 3.6 is no longer supported")
 
 # pylint: disable=wrong-import-position
 
-import cloup
-
 from ch_tools import __version__
 from ch_tools.chadmin.cli.chs3_backup_group import chs3_backup_group
 from ch_tools.chadmin.cli.config_command import config_command
@@ -77,7 +79,6 @@
 @cloup.pass_context
 def cli(ctx, format_, settings, timeout, port, debug):
     """ClickHouse administration tool."""
-
     os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
     handlers: List[logging.Handler] = [logging.FileHandler(LOG_FILE)]
     if debug:
@@ -89,15 +90,16 @@ def cli(ctx, format_, settings, timeout, port, debug):
         handlers=handlers,
     )
 
-    timeout_seconds = timeout.total_seconds() if timeout else None
-    settings = {item[0]: item[1] for item in settings}
+    config = load_config()
+    if port:
+        config["clickhouse"]["port"] = port
+    if timeout:
+        config["clickhouse"]["timeout"] = timeout.total_seconds()
+    if settings:
+        config["clickhouse"]["settings"] = {item[0]: item[1] for item in settings}
 
     ctx.obj = {
-        "chcli_conf": {
-            "port": port,
-            "timeout": timeout_seconds,
-            "settings": settings,
-        },
+        "config": config,
         "format": format_,
         "debug": debug,
     }

ch_tools/chadmin/cli/__init__.py

@@ -3,16 +3,18 @@
 from ch_tools.common.clickhouse.config import ClickhouseConfig
 
 
-def get_config(ctx: Context, try_preprocessed: bool = True) -> ClickhouseConfig:
+def get_clickhouse_config(
+    ctx: Context, try_preprocessed: bool = True
+) -> ClickhouseConfig:
     if "clickhouse_config" not in ctx.obj:
         ctx.obj["clickhouse_config"] = ClickhouseConfig.load(try_preprocessed)
 
     return ctx.obj["clickhouse_config"]
 
 
 def get_macros(ctx):
-    return get_config(ctx).macros
+    return get_clickhouse_config(ctx).macros
 
 
 def get_cluster_name(ctx):
-    return get_config(ctx).cluster_name
+    return get_clickhouse_config(ctx).cluster_name

ch_tools/chadmin/internal/utils.py

@@ -5,29 +5,7 @@
 from itertools import islice
 from typing import Iterable, Iterator
 
-from ch_tools.common.clickhouse.client import ClickhouseClient
-
-
-def clickhouse_client(ctx):
-    """
-    Return ClickHouse client from the context if it exists.
-    Init ClickHouse client and store to the context if it doesn't exist.
-    Raise RuntimeError if ClickHouse client's config doesn't exist
-    """
-    chcli_conf = ctx.obj.get("chcli_conf")
-    if not chcli_conf:
-        raise RuntimeError(
-            "Could not init ClickHouse's connection because there is no chcli config in context. Seems like bug in chadmin."
-        )
-
-    if not ctx.obj.get("chcli"):
-        ctx.obj["chcli"] = ClickhouseClient(
-            port=chcli_conf["port"],
-            timeout=chcli_conf["timeout"],
-            settings=chcli_conf["settings"],
-        )
-
-    return ctx.obj["chcli"]
+from ch_tools.common.clickhouse.client.clickhouse_client import clickhouse_client
 
 
 def execute_query(

ch_tools/chadmin/internal/zookeeper.py

@@ -7,7 +7,7 @@
 from kazoo.client import KazooClient
 from kazoo.exceptions import NoNodeError, NotEmptyError
 
-from ch_tools.chadmin.cli import get_config, get_macros
+from ch_tools.chadmin.cli import get_clickhouse_config, get_macros
 
 
 def get_zk_node(ctx, path, binary=False):
@@ -316,7 +316,7 @@ def _get_zk_client(ctx):
     else:
         # Intentionally don't try to load preprocessed.
         # We are not sure here if zookeeper-servers's changes already have been reloaded by CH.
-        zk_config = get_config(ctx, try_preprocessed=False).zookeeper
+        zk_config = get_clickhouse_config(ctx, try_preprocessed=False).zookeeper
         connect_str = ",".join(
             f'{host if host else node["host"]}:{port if port else node["port"]}'
             for node in zk_config.nodes

ch_tools/common/cli/formatting.py

@@ -23,8 +23,8 @@
 from tabulate import tabulate
 from termcolor import colored
 
+from ..yaml import dump_yaml
 from .utils import get_timezone
-from .yaml import dump_yaml
 
 
 class FormatStyle(Style):

ch_tools/common/clickhouse/client/clickhouse_client.py

@@ -1,5 +1,4 @@
 import logging
-import socket
 from datetime import timedelta
 from typing import Any, Dict, Optional
 
@@ -22,22 +21,19 @@ class ClickhouseClient:
     def __init__(
         self,
         *,
-        host=socket.getfqdn(),
-        port=None,
-        user="_admin",
-        settings=None,
-        timeout=None,
-        insecure=False,
+        host,
+        protocol,
+        insecure,
+        port,
+        user,
+        password,
+        timeout,
+        settings,
     ):
-        if port is None:
-            port = 8443
-        if timeout is None:
-            timeout = 60
-        if settings is None:
-            settings = {}
-
-        self._session = self._create_session(user=user, insecure=insecure)
-        self._url = f"https://{host}:{port}"
+        self._session = self._create_session(
+            user=user, password=password, insecure=insecure
+        )
+        self._url = f"{protocol}://{host}:{port}"
         self._settings = settings
         self._timeout = timeout
         self._ch_version = None
@@ -120,12 +116,42 @@ def render_query(self, query, **kwargs):
         return template.render(kwargs)
 
     @staticmethod
-    def _create_session(user, insecure):
+    def _create_session(user, password, insecure):
         session = requests.Session()
 
         session.verify = False if insecure else "/etc/clickhouse-server/ssl/allCAs.pem"
 
         if user:
             session.headers["X-ClickHouse-User"] = user
+        if password:
+            session.headers["X-ClickHouse-Key"] = password
 
         return session
+
+
+def clickhouse_client(ctx):
+    """
+    Return ClickHouse client from the context if it exists.
+    Init ClickHouse client and store to the context if it doesn't exist.
+    """
+    if not ctx.obj.get("chcli"):
+        config = ctx.obj["config"]["clickhouse"]
+
+        user = config["user"]
+        password = config["password"]
+        if ctx.obj.get("monitoring", False) and config["monitoring_user"]:
+            user = config["monitoring_user"]
+            password = config["monitoring_password"]
+
+        ctx.obj["chcli"] = ClickhouseClient(
+            host=config["host"],
+            protocol=config["protocol"],
+            insecure=config["insecure"],
+            port=config["port"],
+            user=user,
+            password=password,
+            timeout=config["timeout"],
+            settings=config["settings"],
+        )
+
+    return ctx.obj["chcli"]

ch_tools/common/config.py

@@ -0,0 +1,35 @@
+import os.path
+import socket
+from copy import deepcopy
+
+from ch_tools.common.utils import deep_merge
+from ch_tools.common.yaml import load_yaml
+
+CONFIG_FILE = "/etc/clickhouse-tools/config.yaml"
+DEFAULT_CONFIG = {
+    "clickhouse": {
+        "host": socket.getfqdn(),
+        "protocol": "https",
+        "insecure": False,
+        "port": 8443,
+        "user": None,
+        "password": None,
+        "timeout": 60,
+        "settings": {},
+        "monitoring_user": None,
+        "monitoring_password": None,
+    },
+}
+
+
+def load_config():
+    """
+    Read config file, apply defaults and return result configuration.
+    """
+    config = deepcopy(DEFAULT_CONFIG)
+
+    if os.path.exists(CONFIG_FILE):
+        loaded_config = load_yaml(CONFIG_FILE)
+        deep_merge(config, loaded_config)
+
+    return config

ch_tools/monrun_checks/ch_backup.py

@@ -9,21 +9,22 @@
 from os.path import exists
 from typing import Dict, List
 
-import click
+from click import Context
+from cloup import command, option, pass_context
 from dateutil.parser import parse as dateutil_parse
 
 from ch_tools.common.backup import get_backups
 from ch_tools.common.cli.parameters import TimeSpanParamType
-from ch_tools.common.clickhouse.client import ClickhouseClient
+from ch_tools.common.clickhouse.client.clickhouse_client import clickhouse_client
 from ch_tools.common.result import CRIT, OK, WARNING, Result
 
 LOAD_MONITOR_FLAG_PATH = "/tmp/load-monitor-userfault.flag"
 RESTORE_CONTEXT_PATH = "/tmp/ch_backup_restore_state.json"
 FAILED_PARTS_THRESHOLD = 10
 
 
-@click.command("backup")
-@click.option(
+@command("backup")
+@option(
     "--failed-backup-count-crit",
     "--failed-backup-count-crit-threshold",
     "--critical-failed",  # deprecated option name preserved for backward-compatibility
@@ -32,7 +33,7 @@
     help="Critical threshold on the number of failed backups. If last backups failed and its count equals or greater "
     "than the specified threshold, a crit will be reported.",
 )
-@click.option(
+@option(
     "--backup-age-warn",
     "--backup-age-warn-threshold",
     "backup_age_warn_threshold",
@@ -41,7 +42,7 @@
     help="Warning threshold on age of the last backup. If the last backup is created more than the specified "
     "time ago, a warning will be reported.",
 )
-@click.option(
+@option(
     "--backup-age-crit",
     "--backup-age-crit-threshold",
     "--critical-absent",  # deprecated option name preserved for backward-compatibility
@@ -51,22 +52,24 @@
     help="Critical threshold on age of the last backup. If the last backup is created more than the specified "
     "time ago, a crit will be reported.",
 )
-@click.option(
+@option(
     "--backup-count-warn",
     "--backup-count-warn-threshold",
     "backup_count_warn_threshold",
     default=0,
     help="Warning threshold on the number of backups. If the number of backups of any status equals or greater than "
     "the specified threshold, a warning will be reported.",
 )
-@click.option(
+@option(
     "--min-uptime",
     "min_uptime",
     default="2d",
     type=TimeSpanParamType(),
     help="Minimal ClickHouse uptime enough for backup creation.",
 )
+@pass_context
 def backup_command(
+    ctx,
     failed_backup_count_crit_threshold,
     backup_age_warn_threshold,
     backup_age_crit_threshold,
@@ -89,7 +92,7 @@ def backup_command(
             _check_restored_data(),
         )
 
-    _suppress_if_required(result, min_uptime)
+    _suppress_if_required(ctx, result, min_uptime)
 
     return result
 
@@ -205,22 +208,22 @@ def _check_restored_data() -> Result:
     return Result(OK)
 
 
-def _suppress_if_required(result: Result, min_uptime: timedelta) -> None:
+def _suppress_if_required(ctx: Context, result: Result, min_uptime: timedelta) -> None:
     if result.code == OK:
         return
 
     if os.path.exists(LOAD_MONITOR_FLAG_PATH):
         result.code = WARNING
         result.message += " (suppressed by load monitor flag file)"
 
-    if _get_uptime() < min_uptime:
+    if _get_uptime(ctx) < min_uptime:
         result.code = WARNING
         result.message += " (suppressed by low ClickHouse uptime)"
 
 
-def _get_uptime() -> timedelta:
+def _get_uptime(ctx: Context) -> timedelta:
     try:
-        return ClickhouseClient().get_uptime()
+        return clickhouse_client(ctx).get_uptime()
     except Exception:
         logging.warning("Failed to get ClickHouse uptime", exc_info=True)
         return timedelta()