diff --git a/ch_tools/__init__.py b/ch_tools/__init__.py index 8526f52a..69866380 100644 --- a/ch_tools/__init__.py +++ b/ch_tools/__init__.py @@ -1,3 +1,3 @@ """A set of tools for administration and diagnostics of ClickHouse DBMS.""" -__version__ = "1.0.0" +__version__ = "2.548.182050718" diff --git a/ch_tools/chadmin/cli/object_storage_group.py b/ch_tools/chadmin/cli/object_storage_group.py index 5572e441..09ceead4 100644 --- a/ch_tools/chadmin/cli/object_storage_group.py +++ b/ch_tools/chadmin/cli/object_storage_group.py @@ -188,11 +188,7 @@ def _clean_object_storage( ) as resp: # make generator for lazy iterating paths_to_delete = (line.decode() for line in resp.iter_lines()) - if dry_run: - # just count items - deleted = sum(1 for _ in paths_to_delete) - else: - deleted = cleanup_s3_object_storage(disk_conf, paths_to_delete) + deleted = cleanup_s3_object_storage(disk_conf, paths_to_delete, dry_run) click.echo( f"{'Would delete' if dry_run else 'Deleted'} {deleted} objects from bucket [{disk_conf.bucket_name}] with prefix {disk_conf.prefix}" diff --git a/ch_tools/chadmin/internal/object_storage/s3_cleanup.py b/ch_tools/chadmin/internal/object_storage/s3_cleanup.py index 362e09fa..0bb40de7 100644 --- a/ch_tools/chadmin/internal/object_storage/s3_cleanup.py +++ b/ch_tools/chadmin/internal/object_storage/s3_cleanup.py @@ -6,10 +6,12 @@ from ch_tools.chadmin.internal.utils import chunked from ch_tools.common.clickhouse.config.storage_configuration import S3DiskConfiguration -BULK_DELETE_CHUNK_SIZE = 1000 +BULK_DELETE_CHUNK_SIZE = 100 -def cleanup_s3_object_storage(disk: S3DiskConfiguration, keys: Iterator[str]) -> int: +def cleanup_s3_object_storage( + disk: S3DiskConfiguration, keys: Iterator[str], dry_run: bool = False +) -> int: s3 = boto3.resource( "s3", endpoint_url=disk.endpoint_url, @@ -21,7 +23,8 @@ def cleanup_s3_object_storage(disk: S3DiskConfiguration, keys: Iterator[str]) -> deleted = 0 for chunk in chunked(keys, BULK_DELETE_CHUNK_SIZE): - _bulk_delete(bucket, chunk) + if not dry_run: + _bulk_delete(bucket, chunk) deleted += len(chunk) return deleted diff --git a/pyproject.toml b/pyproject.toml index a2df0f77..7e9aa2b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "clickhouse-tools" -version = "0.1.0" +version = "2.548.182050718" license = "MIT" description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS."