From d1091174836b6ea89f94d4aa0b6e5fcfde65dd6f Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Wed, 31 Jul 2024 08:46:34 -0400 Subject: [PATCH] temp: Testing fixes for OOM ClickHouse errors --- xapi_db_load/backends/clickhouse_lake.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/xapi_db_load/backends/clickhouse_lake.py b/xapi_db_load/backends/clickhouse_lake.py index 0ec6fcc..22ef009 100644 --- a/xapi_db_load/backends/clickhouse_lake.py +++ b/xapi_db_load/backends/clickhouse_lake.py @@ -270,10 +270,13 @@ def load_from_s3(self, s3_location): sql = f""" INSERT INTO {table_name} - SELECT * - FROM s3('{file_path}', '{self.s3_key}', '{self.s3_secret}', 'CSV'); + SELECT * + FROM s3('{file_path}', '{self.s3_key}', '{self.s3_secret}', 'CSV') + SETTINGS max_insert_threads = 1, + min_insert_block_size_rows = 524288, + min_insert_block_size_bytes = 134217728; """ - + print(sql) self.client.command(sql) self.print_db_time()