From 7f7fb04e0b405e5abbe959858318a77d71d61d12 Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Mon, 3 Feb 2025 11:02:46 -0500 Subject: [PATCH 1/2] Support separators in doc_id --- hq_superset/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hq_superset/models.py b/hq_superset/models.py index a81d7b7..e9042e0 100644 --- a/hq_superset/models.py +++ b/hq_superset/models.py @@ -35,6 +35,8 @@ class DataSetChange: doc_id: str data: list[dict[str, Any]] + SEP = ',' + def update_dataset(self): with statsd.timed('cca.dataset_change.timer', tags=get_tags({"datasource": self.data_source_id})): self._update_dataset() @@ -66,7 +68,11 @@ def _update_dataset(self): engine.connect() as connection, connection.begin() # Commit on leaving context ): - delete_stmt = table.delete().where(table.c.doc_id == self.doc_id) + if self.SEP in self.doc_id: + doc_ids = self.doc_id.split(self.SEP) + delete_stmt = table.delete().where(table.c.doc_id.in_(doc_ids)) + else: + delete_stmt = table.delete().where(table.c.doc_id == self.doc_id) connection.execute(delete_stmt) if self.data: rows = list(cast_data_for_table(self.data, table)) From 6aa8a65a7f9df0efb64438c6d3771f2a70cd521b Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Tue, 4 Feb 2025 17:26:03 -0500 Subject: [PATCH 2/2] Use new `doc_ids` field --- hq_superset/models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hq_superset/models.py b/hq_superset/models.py index e9042e0..b47a070 100644 --- a/hq_superset/models.py +++ b/hq_superset/models.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any +from typing import Any, Optional from authlib.integrations.sqla_oauth2 import ( OAuth2ClientMixin, @@ -34,8 +34,7 @@ class DataSetChange: data_source_id: str doc_id: str data: list[dict[str, Any]] - - SEP = ',' + doc_ids: Optional[list[str]] def update_dataset(self): with statsd.timed('cca.dataset_change.timer', tags=get_tags({"datasource": self.data_source_id})): @@ -68,9 +67,8 @@ def _update_dataset(self): engine.connect() as connection, connection.begin() # Commit on leaving context ): - if self.SEP in self.doc_id: - doc_ids = self.doc_id.split(self.SEP) - delete_stmt = table.delete().where(table.c.doc_id.in_(doc_ids)) + if self.doc_ids: + delete_stmt = table.delete().where(table.c.doc_id.in_(self.doc_ids)) else: delete_stmt = table.delete().where(table.c.doc_id == self.doc_id) connection.execute(delete_stmt)