Move s3-credentials check command to ch-monitoring (#264)

* Move s3-credentials check command to ch-monitoring

* Update tests

.pylintrc

@@ -312,7 +312,7 @@ max-parents=7
 max-public-methods=25
 
 # Maximum number of return / yield for function / method body.
-max-returns=7
+max-returns=8
 
 # Maximum number of statements in function / method body.
 max-statements=65

ch_tools/chadmin/chadmin_cli.py

@@ -16,7 +16,6 @@
 # pylint: disable=wrong-import-position
 
 from ch_tools import __version__
-from ch_tools.chadmin.cli.ch_s3_credentials_group import ch_s3_credentials_group
 from ch_tools.chadmin.cli.chs3_backup_group import chs3_backup_group
 from ch_tools.chadmin.cli.config_command import config_command
 from ch_tools.chadmin.cli.crash_log_group import crash_log_group
@@ -43,6 +42,7 @@
 from ch_tools.chadmin.cli.replicated_fetch_group import replicated_fetch_group
 from ch_tools.chadmin.cli.replication_queue_group import replication_queue_group
 from ch_tools.chadmin.cli.restore_replica_command import restore_replica_command
+from ch_tools.chadmin.cli.s3_credentials_config_group import s3_credentials_config_group
 from ch_tools.chadmin.cli.stack_trace_command import stack_trace_command
 from ch_tools.chadmin.cli.table_group import table_group
 from ch_tools.chadmin.cli.thread_log_group import thread_log_group
@@ -126,7 +126,7 @@ def cli(ctx, format_, settings, timeout, port, debug):
     object_storage_group,
     part_group,
     part_log_group,
-    ch_s3_credentials_group,
+    s3_credentials_config_group,
     partition_group,
     process_group,
     query_log_group,

ch_tools/chadmin/cli/s3_credentials_config_group.py

@@ -0,0 +1,85 @@
+import json
+import random
+import sys
+import time
+from xml.dom import minidom
+
+import requests
+from click import group, option, pass_context
+
+from ch_tools.chadmin.cli.chadmin_group import Chadmin
+from ch_tools.chadmin.internal.system import match_ch_version
+
+
+@group("s3-credentials-config", cls=Chadmin)
+@option(
+    "-m",
+    "--metadata-address",
+    "metadata_address",
+    default="169.254.169.254",
+    help="Compute metadata api address.",
+)
+@pass_context
+def s3_credentials_config_group(ctx, metadata_address):
+    """
+    Manage default ClickHouse s3 credentials.
+    """
+    ctx.obj["s3_cred_metadata_addr"] = metadata_address
+
+
+def _add_node(document, root, name):
+    node = document.createElement(name)
+    root.appendChild(node)
+    return node
+
+
+def _request_token(endpoint):
+    # pylint: disable=missing-timeout
+    return requests.get(
+        f"http://{endpoint}/computeMetadata/v1/instance/service-accounts/default/token",
+        headers={"Metadata-Flavor": "Google"},
+    )
+
+
+def _get_token(endpoint):
+    response = _request_token(endpoint)
+    if response.status_code != 200:
+        sys.exit(1)
+    data = json.loads(response.content)
+    if data["token_type"] != "Bearer":
+        sys.exit(1)
+    return data["access_token"]
+
+
+@s3_credentials_config_group.command(
+    "update", help="update ch default s3 credentials config"
+)
+@option("-e", "--endpoint", "endpoint", type=str, help="S3 endpoint")
+@option(
+    "-s",
+    "--random-sleep",
+    "random_sleep",
+    default=False,
+    help="whether need a random sleep",
+)
+@pass_context
+def update_s3_credentials(ctx, endpoint, random_sleep):
+    """Update s3 creds."""
+    if random_sleep:
+        time.sleep(random.randint(0, 30))
+
+    doc = minidom.Document()
+    storage = _add_node(
+        doc, _add_node(doc, _add_node(doc, doc, "clickhouse"), "s3"), "cloud_storage"
+    )
+    endpoint_header = (
+        "access_header" if match_ch_version(ctx, min_version="24.11") else "header"
+    )
+    _add_node(doc, storage, "endpoint").appendChild(doc.createTextNode(endpoint))
+    _add_node(doc, storage, endpoint_header).appendChild(
+        doc.createTextNode(
+            f"X-YaCloud-SubjectToken: {_get_token(ctx.obj['s3_cred_metadata_addr'])}"
+        )
+    )
+    with open("/etc/clickhouse-server/config.d/s3_credentials.xml", "wb") as file:
+        file.write(doc.toprettyxml(indent=4 * " ", encoding="utf-8"))

ch_tools/monrun_checks/ch_s3_credentials_config.py

@@ -0,0 +1,80 @@
+import os
+import time
+from datetime import timedelta
+
+import requests
+from click import pass_context
+from cloup import command, option
+
+from ch_tools.common.clickhouse.config.path import (
+    CLICKHOUSE_RESETUP_CONFIG_PATH,
+    CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH,
+)
+from ch_tools.common.result import CRIT, OK, WARNING, Result
+
+
+@command("s3-credentials-config")
+@option(
+    "-p",
+    "--present",
+    default=False,
+    is_flag=True,
+    help="whether config expected to present",
+)
+@pass_context
+def s3_credentials_configs_command(ctx, present):
+    """
+    Check S3 credentials config.
+    """
+    if not present:
+        if not os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH):
+            return Result(OK)
+        return Result(CRIT, "S3 default config present, but shouldn't")
+
+    if os.path.isfile(CLICKHOUSE_RESETUP_CONFIG_PATH):
+        return Result(OK, "Skipped as resetup is in progress")
+
+    if os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH):
+        delta = timedelta(
+            seconds=time.time()
+            - os.path.getmtime(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH)
+        )
+        if delta < timedelta(hours=2):
+            return Result(OK)
+        if delta < timedelta(hours=4):
+            return Result(
+                WARNING,
+                f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours",
+            )
+
+        if delta < timedelta(hours=12):
+            msg = f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours"
+        else:
+            msg = f"S3 token expired {_delta_to_hours(delta - timedelta(hours=12))} hours ago"
+    else:
+        msg = "S3 default config not present"
+
+    code = _request_token(ctx.obj["s3_cred_metadata_addr"]).status_code
+    if code == 404:
+        if "default" in requests.get(
+            f"http://{ctx.obj['s3_cred_metadata_addr']}/computeMetadata/v1/instance/?recursive=true",
+            headers={"Metadata-Flavor": "Google"},
+            timeout=60,
+        ).json().get("serviceAccounts", {}):
+            return Result(WARNING, "service account deleted")
+
+        return Result(CRIT, "service account not linked")
+
+    return Result(CRIT, f"{msg}, iam code {code}")
+
+
+def _request_token(endpoint):
+    # pylint: disable=missing-timeout
+    return requests.get(
+        f"http://{endpoint}/computeMetadata/v1/instance/service-accounts/default/token",
+        headers={"Metadata-Flavor": "Google"},
+    )
+
+
+def _delta_to_hours(delta: timedelta) -> str:
+    return f"{(delta.total_seconds() / 3600):.2f}"

ch_tools/monrun_checks/main.py

@@ -33,6 +33,9 @@
 from ch_tools.monrun_checks.ch_resetup_state import resetup_state_command
 from ch_tools.monrun_checks.ch_ro_replica import ro_replica_command
 from ch_tools.monrun_checks.ch_s3_backup_orphaned import orphaned_backups_command
+from ch_tools.monrun_checks.ch_s3_credentials_config import (
+    s3_credentials_configs_command,
+)
 from ch_tools.monrun_checks.ch_system_queues import system_queues_command
 from ch_tools.monrun_checks.ch_tls import tls_command
 from ch_tools.monrun_checks.dns import dns_command
@@ -150,6 +153,7 @@ def cli(ctx, ensure_monitoring_user):
     geobase_command,
     backup_command,
     orphaned_backups_command,
+    s3_credentials_configs_command,
     tls_command,
     keeper_command,
     dns_command,

tests/features/s3_credentials.feature

@@ -11,19 +11,19 @@ Feature: ch_s3_credentials tool
     Scenario Outline:: chadmin s3 check work correctly
     When we execute command on clickhouse01
     """
-    chadmin ch-s3-credentials check
+    ch-monitoring s3-credentials-config
     """
     Then we get response
     """
     0;OK
     """
     When we execute command on clickhouse01
     """
-    chadmin ch-s3-credentials --metadata-address=http_mock01:8080 update --endpoint=storage.com
+    chadmin s3-credentials-config --metadata-address=http_mock01:8080 update --endpoint=storage.com
     """
     And we execute command on clickhouse01
     """
-    chadmin ch-s3-credentials check --present
+    ch-monitoring s3-credentials-config --present
     """
     Then we get response
     """