From 4b72bedefdf3839cf41c9721528e4c6ec53f4ed0 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 12:17:59 +0300 Subject: [PATCH 01/12] Added regular pattern for handle errors in orphaned-objects monrun check --- ch_tools/monrun_checks/ch_orphaned_objects.py | 4 ++++ tests/features/monrun.feature | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index 959e2b8f..be4eb97d 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -1,4 +1,5 @@ import click +import re from ch_tools.chadmin.cli.object_storage_group import STATE_LOCAL_PATH from ch_tools.chadmin.internal.object_storage.orphaned_objects_state import ( @@ -53,6 +54,9 @@ def orphaned_objects_command( total_size = state.orphaned_objects_size error_msg = state.error_msg + + pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" + error_msg = re.sub(pattern, r"...", error_msg) if error_msg != "": return Result(CRIT, error_msg) diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index b96db5f5..0a0a127c 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -504,7 +504,7 @@ Feature: ch-monitoring tool """ Scenario: Check clickhouse orphaned objects with not empty error_msg - When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"ERROR\" }" + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" And we execute command on clickhouse01 """ ch-monitoring orphaned-objects --state-local From 66172543cb2939bc287b5bca7fb506e9c04a5f4e Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 12:22:22 +0300 Subject: [PATCH 02/12] Update tests --- tests/features/monrun.feature | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index 0a0a127c..d079ae70 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -504,7 +504,7 @@ Feature: ch-monitoring tool """ Scenario: Check clickhouse orphaned objects with not empty error_msg - When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"ERROR\" }" And we execute command on clickhouse01 """ ch-monitoring orphaned-objects --state-local @@ -513,3 +513,14 @@ Feature: ch-monitoring tool """ 2;ERROR """ + + Scenario: Check clickhouse orphaned objects with long error_msg + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" + And we execute command on clickhouse01 + """ + ch-monitoring orphaned-objects --state-local + """ + Then we get response + """ + 2;27. DB::Exception: Cannot parse: input:: expected '\\t' before: (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) + """ From 437b88ca57c6b88aeae48e456312e616f8c86bd7 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 12:24:09 +0300 Subject: [PATCH 03/12] lint --- ch_tools/monrun_checks/ch_orphaned_objects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index be4eb97d..7d435c4f 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -1,6 +1,7 @@ -import click import re +import click + from ch_tools.chadmin.cli.object_storage_group import STATE_LOCAL_PATH from ch_tools.chadmin.internal.object_storage.orphaned_objects_state import ( OrphanedObjectsState, @@ -54,7 +55,7 @@ def orphaned_objects_command( total_size = state.orphaned_objects_size error_msg = state.error_msg - + pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" error_msg = re.sub(pattern, r"...", error_msg) From 4040ce6d13e41c4a47b761c15ea9de76455437bc Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 12:57:56 +0300 Subject: [PATCH 04/12] Refactoring and add spare_pattern --- ch_tools/monrun_checks/ch_orphaned_objects.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index 7d435c4f..68e2cf9a 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -54,10 +54,7 @@ def orphaned_objects_command( return Result(CRIT, str(e)) total_size = state.orphaned_objects_size - error_msg = state.error_msg - - pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" - error_msg = re.sub(pattern, r"...", error_msg) + error_msg = _error_message_format(state.error_msg) if error_msg != "": return Result(CRIT, error_msg) @@ -105,3 +102,13 @@ def _zk_get_orphaned_objects_state( ) -> "OrphanedObjectsState": zk_data = get_zk_node(ctx, state_zk_path) return OrphanedObjectsState.from_json(zk_data) + + +def _error_message_format(error_msg: str) -> str: + main_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" + spare_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*" + if re.match(main_pattern, error_msg): + error_msg = re.search(main_pattern, error_msg) + elif re.match(spare_pattern, error_msg): + error_msg = re.search(spare_pattern, error_msg) + return error_msg From 79540d2671336289948f189acb32482e91f9bfd7 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 13:13:48 +0300 Subject: [PATCH 05/12] Fix test --- tests/features/monrun.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index d079ae70..06e7fda9 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -515,12 +515,12 @@ Feature: ch-monitoring tool """ Scenario: Check clickhouse orphaned objects with long error_msg - When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" And we execute command on clickhouse01 """ ch-monitoring orphaned-objects --state-local """ Then we get response """ - 2;27. DB::Exception: Cannot parse: input:: expected '\\t' before: (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) + 2;Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) """ From 8661cc85621f0d9989de6a5d357a04666aafc472 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 13:19:19 +0300 Subject: [PATCH 06/12] Fix --- ch_tools/monrun_checks/ch_orphaned_objects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index 68e2cf9a..a6a53cc9 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -105,10 +105,10 @@ def _zk_get_orphaned_objects_state( def _error_message_format(error_msg: str) -> str: - main_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" + main_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" spare_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*" if re.match(main_pattern, error_msg): - error_msg = re.search(main_pattern, error_msg) + error_msg = re.sub(main_pattern, r"\1...\2", error_msg) elif re.match(spare_pattern, error_msg): - error_msg = re.search(spare_pattern, error_msg) + error_msg = re.sub(main_pattern, r"\1...", error_msg) return error_msg From 5aac2ebc3695bfc91478088017793d07abcbdb10 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 13:20:56 +0300 Subject: [PATCH 07/12] Fix tests --- tests/features/monrun.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index 06e7fda9..1605bfd0 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -515,12 +515,12 @@ Feature: ch-monitoring tool """ Scenario: Check clickhouse orphaned objects with long error_msg - When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" And we execute command on clickhouse01 """ ch-monitoring orphaned-objects --state-local """ Then we get response """ - 2;Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: (SOME_OTHER_ERROR_CODE) (version 24.8.5.115 (official build)) + 2;Code: 27. DB::Exception: ... (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) """ From b8f23b8970c3dc974f7f0dfdb52c6cef8e855064 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 13:27:32 +0300 Subject: [PATCH 08/12] Create consts --- ch_tools/monrun_checks/ch_orphaned_objects.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index a6a53cc9..0f00b566 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -9,6 +9,9 @@ from ch_tools.chadmin.internal.zookeeper import get_zk_node from ch_tools.common.result import CRIT, OK, WARNING, Result +MAIN_ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" +SPARE_ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*" + @click.command("orphaned-objects") @click.option( @@ -105,10 +108,8 @@ def _zk_get_orphaned_objects_state( def _error_message_format(error_msg: str) -> str: - main_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" - spare_pattern = r"(Code:\s\d+\.\sDB::Exception:\s).*" - if re.match(main_pattern, error_msg): - error_msg = re.sub(main_pattern, r"\1...\2", error_msg) - elif re.match(spare_pattern, error_msg): - error_msg = re.sub(main_pattern, r"\1...", error_msg) + if re.match(MAIN_ERROR_MESSAGE_PATTERN, error_msg): + error_msg = re.sub(MAIN_ERROR_MESSAGE_PATTERN, r"\1...\2", error_msg) + elif re.match(SPARE_ERROR_MESSAGE_PATTERN, error_msg): + error_msg = re.sub(SPARE_ERROR_MESSAGE_PATTERN, r"\1...", error_msg) return error_msg From 164054b0e562d4ae3c8ae5de98727bb5d8a51220 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 13:28:37 +0300 Subject: [PATCH 09/12] Delete SPARE_ERROR_MESSAGE_PATTERN --- ch_tools/monrun_checks/ch_orphaned_objects.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index 0f00b566..c0b4030c 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -9,8 +9,7 @@ from ch_tools.chadmin.internal.zookeeper import get_zk_node from ch_tools.common.result import CRIT, OK, WARNING, Result -MAIN_ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" -SPARE_ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*" +ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" @click.command("orphaned-objects") @@ -108,8 +107,5 @@ def _zk_get_orphaned_objects_state( def _error_message_format(error_msg: str) -> str: - if re.match(MAIN_ERROR_MESSAGE_PATTERN, error_msg): - error_msg = re.sub(MAIN_ERROR_MESSAGE_PATTERN, r"\1...\2", error_msg) - elif re.match(SPARE_ERROR_MESSAGE_PATTERN, error_msg): - error_msg = re.sub(SPARE_ERROR_MESSAGE_PATTERN, r"\1...", error_msg) + error_msg = re.sub(ERROR_MESSAGE_PATTERN, r"\1...\2", error_msg) return error_msg From 4b1fd6ca4225ae2b85f8012acc89646be18d2f4f Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Thu, 5 Dec 2024 16:24:06 +0300 Subject: [PATCH 10/12] Fixes --- ch_tools/monrun_checks/ch_orphaned_objects.py | 4 ++-- tests/features/monrun.feature | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index c0b4030c..c16cc43c 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -9,7 +9,7 @@ from ch_tools.chadmin.internal.zookeeper import get_zk_node from ch_tools.common.result import CRIT, OK, WARNING, Result -ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)).*" +ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)\s).*" @click.command("orphaned-objects") @@ -107,5 +107,5 @@ def _zk_get_orphaned_objects_state( def _error_message_format(error_msg: str) -> str: - error_msg = re.sub(ERROR_MESSAGE_PATTERN, r"\1...\2", error_msg) + error_msg = re.sub(ERROR_MESSAGE_PATTERN, r"\1...\2...", error_msg) return error_msg diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index 1605bfd0..7f1e39bd 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -515,12 +515,12 @@ Feature: ch-monitoring tool """ Scenario: Check clickhouse orphaned objects with long error_msg - When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"Code: 27. DB::Exception: Cannot parse: input:: expected '\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\ndefault\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\n': While executing SystemRemoteDataPaths. (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" + When we create file /tmp/object_storage_cleanup_state.json with data "{ \"orphaned_objects_size\": 0, \"error_msg\": \"Code: 27. DB::Exception: Cannot parse: input:: expected '\\\\t' before: 'klg%2D1acvr8hmq0n16qm5%2Edb%2Eyandex%2Enet\\\\ndefault\\\\n6736d483-516a-4892-87d4-084d5c1f6d3c\\\\n': While executing SystemRemoteDataPaths. (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) Query: SELECT obj_path, obj_size FROM _system.listing_objects_from_object_storage AS object_storage LEFT ANTI JOIN remoteSecure('klg-1acvr8hmq0n16qm5.db.yandex.net', system.remote_data_paths) AS object_table ON object_table.remote_path = object_storage.obj_path AND object_table.disk_name = 'object_storage' SETTINGS traverse_shadow_remote_data_paths=1 FORMAT TabSeparated (klg-1acvr8hmq0n16qm5.mdb.yandex.net)\" }" And we execute command on clickhouse01 """ ch-monitoring orphaned-objects --state-local """ Then we get response """ - 2;Code: 27. DB::Exception: ... (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) + 2;Code: 27. DB::Exception: ... (CANNOT_PARSE_INPUT_ASSERTION_FAILED) (version 24.8.5.115 (official build)) ... """ From 6229422a360fc01eabedd75bc6d110e8c64a51b2 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Fri, 6 Dec 2024 16:10:22 +0300 Subject: [PATCH 11/12] Refactoring, update ch_tools/common/config.py --- ch_tools/common/config.py | 4 ++++ ch_tools/monrun_checks/ch_orphaned_objects.py | 11 +---------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/ch_tools/common/config.py b/ch_tools/common/config.py index cc7d3769..727ec55c 100644 --- a/ch_tools/common/config.py +++ b/ch_tools/common/config.py @@ -84,6 +84,10 @@ "pattern": "\n", "replacement": " ", }, + { + "pattern": r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)\s).*", + "replacement": r"\1...\2...", + }, ], }, }, diff --git a/ch_tools/monrun_checks/ch_orphaned_objects.py b/ch_tools/monrun_checks/ch_orphaned_objects.py index c16cc43c..959e2b8f 100644 --- a/ch_tools/monrun_checks/ch_orphaned_objects.py +++ b/ch_tools/monrun_checks/ch_orphaned_objects.py @@ -1,5 +1,3 @@ -import re - import click from ch_tools.chadmin.cli.object_storage_group import STATE_LOCAL_PATH @@ -9,8 +7,6 @@ from ch_tools.chadmin.internal.zookeeper import get_zk_node from ch_tools.common.result import CRIT, OK, WARNING, Result -ERROR_MESSAGE_PATTERN = r"(Code:\s\d+\.\sDB::Exception:\s).*(\s\([A-Z_]*\)\s\(version\s.*\s\(official build\)\)\s).*" - @click.command("orphaned-objects") @click.option( @@ -56,7 +52,7 @@ def orphaned_objects_command( return Result(CRIT, str(e)) total_size = state.orphaned_objects_size - error_msg = _error_message_format(state.error_msg) + error_msg = state.error_msg if error_msg != "": return Result(CRIT, error_msg) @@ -104,8 +100,3 @@ def _zk_get_orphaned_objects_state( ) -> "OrphanedObjectsState": zk_data = get_zk_node(ctx, state_zk_path) return OrphanedObjectsState.from_json(zk_data) - - -def _error_message_format(error_msg: str) -> str: - error_msg = re.sub(ERROR_MESSAGE_PATTERN, r"\1...\2...", error_msg) - return error_msg From af66fd80fec286374addb043e9736ffcd2a6ae28 Mon Sep 17 00:00:00 2001 From: Nikita Unisikhin Date: Fri, 6 Dec 2024 18:26:31 +0300 Subject: [PATCH 12/12] Update Check Geobase --- tests/features/monrun.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/monrun.feature b/tests/features/monrun.feature index 7f1e39bd..2c357256 100644 --- a/tests/features/monrun.feature +++ b/tests/features/monrun.feature @@ -98,7 +98,7 @@ Feature: ch-monitoring tool """ Then we get response contains """ - 1;Unknown error: Code: 156. DB::Exception: Embedded dictionaries were not loaded. + 1;Unknown error: Code: 156. DB::Exception: ... (DICTIONARIES_WAS_NOT_LOADED) """ When we execute command on clickhouse01 """