diff --git a/.github/workflows/test_clickhouse_version.yml b/.github/workflows/test_clickhouse_version.yml new file mode 100644 index 00000000..3207ef5b --- /dev/null +++ b/.github/workflows/test_clickhouse_version.yml @@ -0,0 +1,48 @@ +name: test_clickhouse_version + +run-name: ${{ github.workflow }}_${{ inputs.clickhouse_version }}_${{ inputs.id || github.run_number }} + +on: + workflow_dispatch: + inputs: + clickhouse_version: + description: 'ClickHouse version' + required: true + type: string + id: + description: 'Run identifier' + required: false + type: string + default: "" + +jobs: + test_integration: + runs-on: ubuntu-latest + env: + CLICKHOUSE_VERSION: ${{ inputs.clickhouse_version }} + CLICKHOUSE_VERSIONS: ${{ inputs.clickhouse_version }} + NEED_SETUP: true + steps: + - uses: actions/checkout@v3 + - name: Check the input tag to ${{ inputs.clickhouse_version }} + continue-on-error: true + run: | + docker manifest inspect chtools/test-clickhouse:${{ inputs.clickhouse_version }} + echo "NEED_SETUP=false" >> $GITHUB_ENV + - name: login to dockerhub + if: env.NEED_SETUP == 'true' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: build and push necessary images + if: env.NEED_SETUP == 'true' + uses: docker/bake-action@v3 + with: + files: tests/bake.hcl + push: true + - uses: ./.github/actions/setup_dependencies + with: + python-version: "3.11" + - name: run integration tests + run: make test-integration diff --git a/ch_tools/chadmin/cli/database_group.py b/ch_tools/chadmin/cli/database_group.py index b22c4f77..5adb4d52 100644 --- a/ch_tools/chadmin/cli/database_group.py +++ b/ch_tools/chadmin/cli/database_group.py @@ -1,5 +1,6 @@ from click import argument, group, option, pass_context +from ch_tools.chadmin.cli import get_cluster_name from ch_tools.chadmin.internal.utils import execute_query @@ -47,7 +48,13 @@ def list_databases_command(ctx, **kwargs): @option("-d", "--database") @option("--exclude-database") @option("-a", "--all", "all_", is_flag=True, help="Delete all databases.") -@option("--cluster") +@option( + "--cluster", + "--on-cluster", + "on_cluster", + is_flag=True, + help="Delete databases on all hosts of the cluster.", +) @option( "-n", "--dry-run", @@ -55,10 +62,14 @@ def list_databases_command(ctx, **kwargs): default=False, help="Enable dry run mode and do not perform any modifying actions.", ) -def delete_databases_command(ctx, dry_run, all_, database, exclude_database, cluster): +def delete_databases_command( + ctx, dry_run, all_, database, exclude_database, on_cluster +): if not any((all_, database)): ctx.fail("At least one of --all, --database options must be specified.") + cluster = get_cluster_name(ctx) if on_cluster else None + for d in get_databases( ctx, database=database, exclude_database=exclude_database, format_="JSON" )["data"]: diff --git a/ch_tools/chadmin/cli/mutation_group.py b/ch_tools/chadmin/cli/mutation_group.py index 060acfea..79208e1c 100644 --- a/ch_tools/chadmin/cli/mutation_group.py +++ b/ch_tools/chadmin/cli/mutation_group.py @@ -53,8 +53,11 @@ def get_mutation(ctx, mutation, last): is_flag=True, help="Get mutations from all hosts in the cluster.", ) +@option( + "-l", "--limit", type=int, help="Limit the max number of objects in the output." +) @pass_context -def list_mutations(ctx, is_done, command_pattern, on_cluster): +def list_mutations(ctx, is_done, command_pattern, on_cluster, limit): """List mutations.""" cluster = get_cluster_name(ctx) if on_cluster else None query = """ @@ -86,6 +89,9 @@ def list_mutations(ctx, is_done, command_pattern, on_cluster): {% if command_pattern %} AND command ILIKE '{{ command_pattern }}' {% endif %} + {% if limit -%} + LIMIT {{ limit }} + {% endif -%} """ response = execute_query( ctx, @@ -93,6 +99,7 @@ def list_mutations(ctx, is_done, command_pattern, on_cluster): is_done=is_done, command_pattern=command_pattern, cluster=cluster, + limit=limit, format_="Vertical", ) print(response) diff --git a/ch_tools/chadmin/cli/table_group.py b/ch_tools/chadmin/cli/table_group.py index 639be479..7d684b2d 100644 --- a/ch_tools/chadmin/cli/table_group.py +++ b/ch_tools/chadmin/cli/table_group.py @@ -120,7 +120,19 @@ def columns_command(ctx, database, table): "--exclude-table", help="Filter out tables to delete by the specified table name." ) @option("-a", "--all", "all_", is_flag=True, help="Delete all tables.") -@option("--cluster") +@option( + "--cluster", + "--on-cluster", + "on_cluster", + is_flag=True, + help="Delete tables on all hosts of the cluster.", +) +@option( + "--sync/--async", + "sync_mode", + default=True, + help="Enable/Disable synchronous query execution.", +) @option( "-n", "--dry-run", @@ -128,7 +140,9 @@ def columns_command(ctx, database, table): default=False, help="Enable dry run mode and do not perform any modifying actions.", ) -def delete_command(ctx, dry_run, all_, database, table, exclude_table, cluster): +def delete_command( + ctx, all_, database, table, exclude_table, on_cluster, sync_mode, dry_run +): """ Delete one or several tables. """ @@ -137,6 +151,8 @@ def delete_command(ctx, dry_run, all_, database, table, exclude_table, cluster): "At least one of --all, --database, --table options must be specified." ) + cluster = get_cluster_name(ctx) if on_cluster else None + for t in list_tables( ctx, database=database, table=table, exclude_table=exclude_table ): @@ -145,6 +161,7 @@ def delete_command(ctx, dry_run, all_, database, table, exclude_table, cluster): database=t["database"], table=t["table"], cluster=cluster, + sync_mode=sync_mode, echo=True, dry_run=dry_run, ) diff --git a/ch_tools/chadmin/cli/zookeeper_group.py b/ch_tools/chadmin/cli/zookeeper_group.py index 4e99679f..154106c5 100644 --- a/ch_tools/chadmin/cli/zookeeper_group.py +++ b/ch_tools/chadmin/cli/zookeeper_group.py @@ -49,7 +49,6 @@ type=str, help="Cluster ZooKeeper root path. If not specified,the root path will be used.", required=False, - default="/", ) @pass_context def zookeeper_group( diff --git a/ch_tools/chadmin/internal/table.py b/ch_tools/chadmin/internal/table.py index 86567fd2..b38b6b77 100644 --- a/ch_tools/chadmin/internal/table.py +++ b/ch_tools/chadmin/internal/table.py @@ -149,7 +149,9 @@ def attach_table(ctx, database, table, *, cluster=None, echo=False, dry_run=Fals ) -def delete_table(ctx, database, table, *, cluster=None, echo=False, dry_run=False): +def delete_table( + ctx, database, table, *, cluster=None, echo=False, sync_mode=True, dry_run=False +): """ Perform "DROP TABLE" for the specified table. """ @@ -158,7 +160,9 @@ def delete_table(ctx, database, table, *, cluster=None, echo=False, dry_run=Fals {%- if cluster %} ON CLUSTER '{{ cluster }}' {%- endif %} + {%- if sync_mode %} NO DELAY + {%- endif %} """ execute_query( ctx, @@ -166,6 +170,7 @@ def delete_table(ctx, database, table, *, cluster=None, echo=False, dry_run=Fals database=database, table=table, cluster=cluster, + sync_mode=sync_mode, echo=echo, dry_run=dry_run, format_=None, diff --git a/ch_tools/chadmin/internal/zookeeper.py b/ch_tools/chadmin/internal/zookeeper.py index 1790e08f..4c049383 100644 --- a/ch_tools/chadmin/internal/zookeeper.py +++ b/ch_tools/chadmin/internal/zookeeper.py @@ -321,7 +321,7 @@ def _get_zk_client(ctx): f'{host if host else node["host"]}:{port if port else node["port"]}' for node in zk_config.nodes ) - if not zk_root_path: + if zk_root_path: connect_str += zk_root_path elif not no_chroot and zk_config.root is not None: connect_str += zk_config.root diff --git a/ch_tools/monrun_checks/ext_ip_dns.py b/ch_tools/monrun_checks/ext_ip_dns.py index e4ee565f..aa6a0b41 100644 --- a/ch_tools/monrun_checks/ext_ip_dns.py +++ b/ch_tools/monrun_checks/ext_ip_dns.py @@ -1,7 +1,7 @@ import json import socket from functools import lru_cache -from typing import List +from typing import List, Tuple import click import dns.resolver @@ -56,7 +56,7 @@ def _check_fqdn(target: _TargetRecord, ipv6: bool, imdsv2: bool) -> list: err = [] resolver = dns.resolver.Resolver() - def _compare(record_type: str, ip_type: str) -> bool: + def _compare(record_type: str, ip_type: str) -> Tuple[bool, set, set]: try: actual_addr = set( map(lambda a: a.to_text(), resolver.resolve(target.fqdn, record_type)) @@ -65,13 +65,21 @@ def _compare(record_type: str, ip_type: str) -> bool: actual_addr = set() target_addr = {_get_host_ip(ip_type, imdsv2)} if target.strict: - return target_addr == actual_addr - return actual_addr.issuperset(target_addr) - - if not _compare("A", "private_v4" if target.private else "public_v4"): - err.append(f"{target.fqdn}: invalid A") - if ipv6 and not _compare("AAAA", "ipv6"): - err.append(f"{target.fqdn}: invalid AAAA") + return target_addr == actual_addr, target_addr, actual_addr + return actual_addr.issuperset(target_addr), target_addr, actual_addr + + ok, target_addr, actual_addr = _compare( + "A", "private_v4" if target.private else "public_v4" + ) + if not ok: + err.append( + f"{target.fqdn}: invalid A: expected {target_addr}, actual {actual_addr}" + ) + ok, target_addr, actual_addr = _compare("AAAA", "ipv6") + if ipv6 and not ok: + err.append( + f"{target.fqdn}: invalid AAAA: expected {target_addr}, actual {actual_addr}" + ) return err diff --git a/tests/features/chadmin_zookeeper.feature b/tests/features/chadmin_zookeeper.feature index f93df3f0..702bcd92 100644 --- a/tests/features/chadmin_zookeeper.feature +++ b/tests/features/chadmin_zookeeper.feature @@ -1,4 +1,4 @@ -Feature: keeper-monitoring tool +Feature: chadmin zookeeper commands. Background: Given default configuration