temp: Testing fixes for OOM ClickHouse errors

openedx · Jul 31, 2024 · d109117 · d109117
1 parent e64b602
commit d109117
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/xapi_db_load/backends/clickhouse_lake.py b/xapi_db_load/backends/clickhouse_lake.py
@@ -270,10 +270,13 @@ def load_from_s3(self, s3_location):
 
             sql = f"""
             INSERT INTO {table_name}
-               SELECT *
-               FROM s3('{file_path}', '{self.s3_key}', '{self.s3_secret}', 'CSV');
+                SELECT *
+                FROM s3('{file_path}', '{self.s3_key}', '{self.s3_secret}', 'CSV')
+            SETTINGS max_insert_threads = 1,
+                min_insert_block_size_rows = 524288,
+                min_insert_block_size_bytes = 134217728;
             """
-
+            print(sql)
             self.client.command(sql)
             self.print_db_time()