diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.py index 616d1b1d8e57f..ddc5901a637d8 100644 --- a/posthog/hogql/database/test/test_database.py +++ b/posthog/hogql/database/test/test_database.py @@ -111,6 +111,27 @@ def test_serialize_database_warehouse_table_s3(self): assert field.type == "string" assert field.schema_valid is True + def test_serialize_database_warehouse_with_deleted_joins(self): + DataWarehouseJoin.objects.create( + team=self.team, + source_table_name="events", + source_table_key="event", + joining_table_name="groups", + joining_table_key="key", + field_name="some_field", + deleted=True, + ) + + db = create_hogql_database(team_id=self.team.pk) + + serialized_database = serialize_database(HogQLContext(team_id=self.team.pk, database=db)) + + events_table = serialized_database.get("events") + assert events_table is not None + + joined_field = events_table.fields.get("some_field") + assert joined_field is None + def test_serialize_database_warehouse_table_s3_with_hyphens(self): credentials = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team) DataWarehouseTable.objects.create( diff --git a/posthog/warehouse/api/saved_query.py b/posthog/warehouse/api/saved_query.py index ac8c8e53022dd..2d8ef156aa6b0 100644 --- a/posthog/warehouse/api/saved_query.py +++ b/posthog/warehouse/api/saved_query.py @@ -4,6 +4,7 @@ import structlog from asgiref.sync import async_to_sync from django.db import transaction +from django.db.models import Q from rest_framework import exceptions, filters, request, response, serializers, status, viewsets from rest_framework.decorators import action @@ -155,8 +156,11 @@ def safely_get_queryset(self, queryset): def destroy(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response: instance: DataWarehouseSavedQuery = self.get_object() - DataWarehouseJoin.objects.filter(source_table_name=instance.name).delete() - DataWarehouseJoin.objects.filter(joining_table_name=instance.name).delete() + + for join in DataWarehouseJoin.objects.filter( + Q(team_id=instance.team_id) & (Q(source_table_name=instance.name) | Q(joining_table_name=instance.name)) + ).exclude(deleted=True): + join.soft_delete() if instance.table is not None: instance.table.soft_delete() diff --git a/posthog/warehouse/data_load/source_templates.py b/posthog/warehouse/data_load/source_templates.py index 17dd1b7b9fecd..5a7d515bc8536 100644 --- a/posthog/warehouse/data_load/source_templates.py +++ b/posthog/warehouse/data_load/source_templates.py @@ -7,23 +7,31 @@ @database_sync_to_async def database_operations(team_id: int, table_prefix: str) -> None: - customer_join_exists = DataWarehouseJoin.objects.filter( - team_id=team_id, - source_table_name="persons", - source_table_key="properties.email", - joining_table_name=f"{table_prefix}stripe_customer", - joining_table_key="email", - field_name=f"{table_prefix}stripe_customer", - ).exists() + customer_join_exists = ( + DataWarehouseJoin.objects.filter( + team_id=team_id, + source_table_name="persons", + source_table_key="properties.email", + joining_table_name=f"{table_prefix}stripe_customer", + joining_table_key="email", + field_name=f"{table_prefix}stripe_customer", + ) + .exclude(deleted=True) + .exists() + ) - invoice_join_exists = DataWarehouseJoin.objects.filter( - team_id=team_id, - source_table_name="persons", - source_table_key="properties.email", - joining_table_name=f"{table_prefix}stripe_invoice", - joining_table_key="customer_email", - field_name=f"{table_prefix}stripe_invoice", - ).exists() + invoice_join_exists = ( + DataWarehouseJoin.objects.filter( + team_id=team_id, + source_table_name="persons", + source_table_key="properties.email", + joining_table_name=f"{table_prefix}stripe_invoice", + joining_table_key="customer_email", + field_name=f"{table_prefix}stripe_invoice", + ) + .exclude(deleted=True) + .exists() + ) if not customer_join_exists: DataWarehouseJoin.objects.create( diff --git a/posthog/warehouse/models/join.py b/posthog/warehouse/models/join.py index 000b1ba34f9b2..febbf0182f1ca 100644 --- a/posthog/warehouse/models/join.py +++ b/posthog/warehouse/models/join.py @@ -1,6 +1,6 @@ from typing import Optional from warnings import warn - +from datetime import datetime from django.db import models from posthog.hogql.ast import SelectQuery @@ -41,6 +41,11 @@ class DataWarehouseJoin(CreatedMetaFields, UUIDModel, DeletedMetaFields): joining_table_key = models.CharField(max_length=400) field_name = models.CharField(max_length=400) + def soft_delete(self): + self.deleted = True + self.deleted_at = datetime.now() + self.save() + def join_function( self, override_source_table_key: Optional[str] = None, override_joining_table_key: Optional[str] = None ): diff --git a/posthog/warehouse/models/table.py b/posthog/warehouse/models/table.py index 928ca1596456c..a83cf2735f56e 100644 --- a/posthog/warehouse/models/table.py +++ b/posthog/warehouse/models/table.py @@ -104,8 +104,10 @@ class TableFormat(models.TextChoices): def soft_delete(self): from posthog.warehouse.models.join import DataWarehouseJoin - DataWarehouseJoin.objects.filter(source_table_name=self.name).delete() - DataWarehouseJoin.objects.filter(joining_table_name=self.name).delete() + for join in DataWarehouseJoin.objects.filter( + Q(team_id=self.team.pk) & (Q(source_table_name=self.name) | Q(joining_table_name=self.name)) + ).exclude(deleted=True): + join.soft_delete() self.deleted = True self.deleted_at = datetime.now()