Refactor dry_run handling

yandex · Jan 30, 2024 · 5c88cf2 · 5c88cf2
1 parent 252e3de
commit 5c88cf2
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 10 deletions.
diff --git a/ch_tools/__init__.py b/ch_tools/__init__.py
@@ -1,3 +1,3 @@
 """A set of tools for administration and diagnostics of ClickHouse DBMS."""
 
-__version__ = "1.0.0"
+__version__ = "2.548.182050718"
diff --git a/ch_tools/chadmin/cli/object_storage_group.py b/ch_tools/chadmin/cli/object_storage_group.py
@@ -188,11 +188,7 @@ def _clean_object_storage(
     ) as resp:
         # make generator for lazy iterating
         paths_to_delete = (line.decode() for line in resp.iter_lines())
-        if dry_run:
-            # just count items
-            deleted = sum(1 for _ in paths_to_delete)
-        else:
-            deleted = cleanup_s3_object_storage(disk_conf, paths_to_delete)
+        deleted = cleanup_s3_object_storage(disk_conf, paths_to_delete, dry_run)
 
     click.echo(
         f"{'Would delete' if dry_run else 'Deleted'} {deleted} objects from bucket [{disk_conf.bucket_name}] with prefix {disk_conf.prefix}"

diff --git a/ch_tools/chadmin/internal/object_storage/s3_cleanup.py b/ch_tools/chadmin/internal/object_storage/s3_cleanup.py
@@ -6,10 +6,12 @@
 from ch_tools.chadmin.internal.utils import chunked
 from ch_tools.common.clickhouse.config.storage_configuration import S3DiskConfiguration
 
-BULK_DELETE_CHUNK_SIZE = 1000
+BULK_DELETE_CHUNK_SIZE = 100
 
 
-def cleanup_s3_object_storage(disk: S3DiskConfiguration, keys: Iterator[str]) -> int:
+def cleanup_s3_object_storage(
+    disk: S3DiskConfiguration, keys: Iterator[str], dry_run: bool = False
+) -> int:
     s3 = boto3.resource(
         "s3",
         endpoint_url=disk.endpoint_url,
@@ -21,7 +23,8 @@ def cleanup_s3_object_storage(disk: S3DiskConfiguration, keys: Iterator[str]) ->
     deleted = 0
 
     for chunk in chunked(keys, BULK_DELETE_CHUNK_SIZE):
-        _bulk_delete(bucket, chunk)
+        if not dry_run:
+            _bulk_delete(bucket, chunk)
         deleted += len(chunk)
 
     return deleted

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "clickhouse-tools"
-version = "0.1.0"
+version = "2.548.182050718"
 license = "MIT"
 description = "clickhouse-tools is a set of tools for administration and diagnostics of ClickHouse DBMS."