diff --git a/.pylintrc b/.pylintrc index 1f335f6a..a65831bf 100644 --- a/.pylintrc +++ b/.pylintrc @@ -312,7 +312,7 @@ max-parents=7 max-public-methods=25 # Maximum number of return / yield for function / method body. -max-returns=7 +max-returns=8 # Maximum number of statements in function / method body. max-statements=65 diff --git a/ch_tools/chadmin/chadmin_cli.py b/ch_tools/chadmin/chadmin_cli.py index a8f84c98..ae44436d 100755 --- a/ch_tools/chadmin/chadmin_cli.py +++ b/ch_tools/chadmin/chadmin_cli.py @@ -16,7 +16,6 @@ # pylint: disable=wrong-import-position from ch_tools import __version__ -from ch_tools.chadmin.cli.ch_s3_credentials_group import ch_s3_credentials_group from ch_tools.chadmin.cli.chs3_backup_group import chs3_backup_group from ch_tools.chadmin.cli.config_command import config_command from ch_tools.chadmin.cli.crash_log_group import crash_log_group @@ -43,6 +42,7 @@ from ch_tools.chadmin.cli.replicated_fetch_group import replicated_fetch_group from ch_tools.chadmin.cli.replication_queue_group import replication_queue_group from ch_tools.chadmin.cli.restore_replica_command import restore_replica_command +from ch_tools.chadmin.cli.s3_credentials_config_group import s3_credentials_config_group from ch_tools.chadmin.cli.stack_trace_command import stack_trace_command from ch_tools.chadmin.cli.table_group import table_group from ch_tools.chadmin.cli.thread_log_group import thread_log_group @@ -126,7 +126,7 @@ def cli(ctx, format_, settings, timeout, port, debug): object_storage_group, part_group, part_log_group, - ch_s3_credentials_group, + s3_credentials_config_group, partition_group, process_group, query_log_group, diff --git a/ch_tools/chadmin/cli/ch_s3_credentials_group.py b/ch_tools/chadmin/cli/ch_s3_credentials_group.py deleted file mode 100644 index 64c0cd30..00000000 --- a/ch_tools/chadmin/cli/ch_s3_credentials_group.py +++ /dev/null @@ -1,155 +0,0 @@ -import json -import os.path -import random -import sys -import time -from datetime import timedelta -from xml.dom import minidom - -import requests -from click import group, option, pass_context - -from ch_tools.chadmin.cli.chadmin_group import Chadmin -from ch_tools.chadmin.internal.system import match_ch_version -from ch_tools.common.clickhouse.config.path import ( - CLICKHOUSE_RESETUP_CONFIG_PATH, - CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH, -) - - -@group("ch-s3-credentials", cls=Chadmin) -@option( - "-m", - "--metadata-address", - "metadata_address", - default="169.254.169.254", - help="Compute metadata api address.", -) -@pass_context -def ch_s3_credentials_group(ctx, metadata_address): - """ - Manage default ClickHouse s3 credentials. - """ - ctx.obj["s3_cred_metadata_addr"] = metadata_address - - -def _add_node(document, root, name): - node = document.createElement(name) - root.appendChild(node) - return node - - -def _request_token(endpoint): - # pylint: disable=missing-timeout - return requests.get( - f"http://{endpoint}/computeMetadata/v1/instance/service-accounts/default/token", - headers={"Metadata-Flavor": "Google"}, - ) - - -def _get_token(endpoint): - response = _request_token(endpoint) - if response.status_code != 200: - sys.exit(1) - data = json.loads(response.content) - if data["token_type"] != "Bearer": - sys.exit(1) - return data["access_token"] - - -@ch_s3_credentials_group.command( - "update", help="update ch default s3 credentials config" -) -@option("-e", "--endpoint", "endpoint", type=str, help="S3 endpoint") -@option( - "-s", - "--random-sleep", - "random_sleep", - default=False, - help="whether need a random sleep", -) -@pass_context -def update_s3_credentials(ctx, endpoint, random_sleep): - """Update s3 creds.""" - if random_sleep: - time.sleep(random.randint(0, 30)) - - doc = minidom.Document() - storage = _add_node( - doc, _add_node(doc, _add_node(doc, doc, "clickhouse"), "s3"), "cloud_storage" - ) - endpoint_header = ( - "access_header" if match_ch_version(ctx, min_version="24.11") else "header" - ) - _add_node(doc, storage, "endpoint").appendChild(doc.createTextNode(endpoint)) - _add_node(doc, storage, endpoint_header).appendChild( - doc.createTextNode( - f"X-YaCloud-SubjectToken: {_get_token(ctx.obj['s3_cred_metadata_addr'])}" - ) - ) - with open("/etc/clickhouse-server/config.d/s3_credentials.xml", "wb") as file: - file.write(doc.toprettyxml(indent=4 * " ", encoding="utf-8")) - - -def result(code, msg): - print(f"{code};{msg}") - sys.exit(0) - - -def _delta_to_hours(delta: timedelta) -> str: - return f"{(delta.total_seconds() / 3600):.2f}" - - -@ch_s3_credentials_group.command( - "check", help="check ch default s3 credentials config status" -) -@option( - "-p", - "--present", - default=False, - is_flag=True, - help="whether config expected to present", -) -@pass_context -def check_s3_credentials(ctx, present): - # pylint: disable=missing-timeout - if not present: - if os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH): - result(2, "S3 default config present, but shouldn't") - else: - result(0, "OK") - - if os.path.isfile(CLICKHOUSE_RESETUP_CONFIG_PATH): - result(0, "Skipped as resetup is in progress") - - if os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH): - delta = timedelta( - seconds=time.time() - - os.path.getmtime(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH) - ) - if delta < timedelta(hours=2): - result(0, "OK") - elif delta < timedelta(hours=4): - result( - 1, - f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours", - ) - - if delta < timedelta(hours=12): - msg = f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours" - else: - msg = f"S3 token expired {_delta_to_hours(delta - timedelta(hours=12))} hours ago" - else: - msg = "S3 default config not present" - - code = _request_token(ctx.obj["s3_cred_metadata_addr"]).status_code - if code == 404: - if "default" in requests.get( - f"http://{ctx.obj['s3_cred_metadata_addr']}/computeMetadata/v1/instance/?recursive=true", - headers={"Metadata-Flavor": "Google"}, - ).json().get("serviceAccounts", {}): - result(1, "service account deleted") - else: - result(2, "service account not linked") - - result(2, msg + f", iam code {code}") diff --git a/ch_tools/chadmin/cli/s3_credentials_config_group.py b/ch_tools/chadmin/cli/s3_credentials_config_group.py new file mode 100644 index 00000000..c78919bf --- /dev/null +++ b/ch_tools/chadmin/cli/s3_credentials_config_group.py @@ -0,0 +1,85 @@ +import json +import random +import sys +import time +from xml.dom import minidom + +import requests +from click import group, option, pass_context + +from ch_tools.chadmin.cli.chadmin_group import Chadmin +from ch_tools.chadmin.internal.system import match_ch_version + + +@group("s3-credentials-config", cls=Chadmin) +@option( + "-m", + "--metadata-address", + "metadata_address", + default="169.254.169.254", + help="Compute metadata api address.", +) +@pass_context +def s3_credentials_config_group(ctx, metadata_address): + """ + Manage default ClickHouse s3 credentials. + """ + ctx.obj["s3_cred_metadata_addr"] = metadata_address + + +def _add_node(document, root, name): + node = document.createElement(name) + root.appendChild(node) + return node + + +def _request_token(endpoint): + # pylint: disable=missing-timeout + return requests.get( + f"http://{endpoint}/computeMetadata/v1/instance/service-accounts/default/token", + headers={"Metadata-Flavor": "Google"}, + ) + + +def _get_token(endpoint): + response = _request_token(endpoint) + if response.status_code != 200: + sys.exit(1) + data = json.loads(response.content) + if data["token_type"] != "Bearer": + sys.exit(1) + return data["access_token"] + + +@s3_credentials_config_group.command( + "update", help="update ch default s3 credentials config" +) +@option("-e", "--endpoint", "endpoint", type=str, help="S3 endpoint") +@option( + "-s", + "--random-sleep", + "random_sleep", + default=False, + help="whether need a random sleep", +) +@pass_context +def update_s3_credentials(ctx, endpoint, random_sleep): + """Update s3 creds.""" + if random_sleep: + time.sleep(random.randint(0, 30)) + + doc = minidom.Document() + storage = _add_node( + doc, _add_node(doc, _add_node(doc, doc, "clickhouse"), "s3"), "cloud_storage" + ) + endpoint_header = ( + "access_header" if match_ch_version(ctx, min_version="24.11") else "header" + ) + _add_node(doc, storage, "endpoint").appendChild(doc.createTextNode(endpoint)) + _add_node(doc, storage, endpoint_header).appendChild( + doc.createTextNode( + f"X-YaCloud-SubjectToken: {_get_token(ctx.obj['s3_cred_metadata_addr'])}" + ) + ) + with open("/etc/clickhouse-server/config.d/s3_credentials.xml", "wb") as file: + file.write(doc.toprettyxml(indent=4 * " ", encoding="utf-8")) diff --git a/ch_tools/monrun_checks/ch_s3_credentials_config.py b/ch_tools/monrun_checks/ch_s3_credentials_config.py new file mode 100644 index 00000000..e2336832 --- /dev/null +++ b/ch_tools/monrun_checks/ch_s3_credentials_config.py @@ -0,0 +1,80 @@ +import os +import time +from datetime import timedelta + +import requests +from click import pass_context +from cloup import command, option + +from ch_tools.common.clickhouse.config.path import ( + CLICKHOUSE_RESETUP_CONFIG_PATH, + CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH, +) +from ch_tools.common.result import CRIT, OK, WARNING, Result + + +@command("s3-credentials-config") +@option( + "-p", + "--present", + default=False, + is_flag=True, + help="whether config expected to present", +) +@pass_context +def s3_credentials_configs_command(ctx, present): + """ + Check S3 credentials config. + """ + if not present: + if not os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH): + return Result(OK) + return Result(CRIT, "S3 default config present, but shouldn't") + + if os.path.isfile(CLICKHOUSE_RESETUP_CONFIG_PATH): + return Result(OK, "Skipped as resetup is in progress") + + if os.path.exists(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH): + delta = timedelta( + seconds=time.time() + - os.path.getmtime(CLICKHOUSE_S3_CREDENTIALS_CONFIG_PATH) + ) + if delta < timedelta(hours=2): + return Result(OK) + if delta < timedelta(hours=4): + return Result( + WARNING, + f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours", + ) + + if delta < timedelta(hours=12): + msg = f"S3 token expire in {_delta_to_hours(timedelta(hours=12) - delta)} hours" + else: + msg = f"S3 token expired {_delta_to_hours(delta - timedelta(hours=12))} hours ago" + else: + msg = "S3 default config not present" + + code = _request_token(ctx.obj["s3_cred_metadata_addr"]).status_code + if code == 404: + if "default" in requests.get( + f"http://{ctx.obj['s3_cred_metadata_addr']}/computeMetadata/v1/instance/?recursive=true", + headers={"Metadata-Flavor": "Google"}, + timeout=60, + ).json().get("serviceAccounts", {}): + return Result(WARNING, "service account deleted") + + return Result(CRIT, "service account not linked") + + return Result(CRIT, f"{msg}, iam code {code}") + + +def _request_token(endpoint): + # pylint: disable=missing-timeout + return requests.get( + f"http://{endpoint}/computeMetadata/v1/instance/service-accounts/default/token", + headers={"Metadata-Flavor": "Google"}, + ) + + +def _delta_to_hours(delta: timedelta) -> str: + return f"{(delta.total_seconds() / 3600):.2f}" diff --git a/ch_tools/monrun_checks/main.py b/ch_tools/monrun_checks/main.py index ef762e40..c0b63843 100644 --- a/ch_tools/monrun_checks/main.py +++ b/ch_tools/monrun_checks/main.py @@ -33,6 +33,9 @@ from ch_tools.monrun_checks.ch_resetup_state import resetup_state_command from ch_tools.monrun_checks.ch_ro_replica import ro_replica_command from ch_tools.monrun_checks.ch_s3_backup_orphaned import orphaned_backups_command +from ch_tools.monrun_checks.ch_s3_credentials_config import ( + s3_credentials_configs_command, +) from ch_tools.monrun_checks.ch_system_queues import system_queues_command from ch_tools.monrun_checks.ch_tls import tls_command from ch_tools.monrun_checks.dns import dns_command @@ -150,6 +153,7 @@ def cli(ctx, ensure_monitoring_user): geobase_command, backup_command, orphaned_backups_command, + s3_credentials_configs_command, tls_command, keeper_command, dns_command, diff --git a/tests/features/s3_credentials.feature b/tests/features/s3_credentials.feature index ca2feb05..c804bb6d 100644 --- a/tests/features/s3_credentials.feature +++ b/tests/features/s3_credentials.feature @@ -11,7 +11,7 @@ Feature: ch_s3_credentials tool Scenario Outline:: chadmin s3 check work correctly When we execute command on clickhouse01 """ - chadmin ch-s3-credentials check + ch-monitoring s3-credentials-config """ Then we get response """ @@ -19,11 +19,11 @@ Feature: ch_s3_credentials tool """ When we execute command on clickhouse01 """ - chadmin ch-s3-credentials --metadata-address=http_mock01:8080 update --endpoint=storage.com + chadmin s3-credentials-config --metadata-address=http_mock01:8080 update --endpoint=storage.com """ And we execute command on clickhouse01 """ - chadmin ch-s3-credentials check --present + ch-monitoring s3-credentials-config --present """ Then we get response """