Skip to content

Commit

Permalink
fix(form): add prefetching and fieldset reuse during calc answer eval…
Browse files Browse the repository at this point in the history 
…uation
  • Loading branch information
@luytena
luytena committed Dec 9, 2024
1 parent 8b46766 commit 3f0a010
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 36 deletions.
39 changes: 27 additions & 12 deletions caluma/caluma_form/domain_logic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@

from caluma.caluma_core.models import BaseModel
from caluma.caluma_core.relay import extract_global_id
from caluma.caluma_form import models, validators
from caluma.caluma_form import models, validators, structure, utils
from caluma.caluma_form.utils import recalculate_answers_from_document, update_or_create_calc_answer
from caluma.caluma_user.models import BaseUser
from caluma.utils import update_model




class BaseLogic:
@staticmethod
@transaction.atomic
Expand Down Expand Up @@ -167,11 +169,17 @@ def create(

if answer.question.type == models.Question.TYPE_TABLE:
answer.create_answer_documents(documents)
for question in models.Question.objects.filter(
pk__in=answer.question.calc_dependents
):
print(f"recalculating {question} from domain logic _create_")
update_or_create_calc_answer(question, answer.document)

for question in models.Question.objects.filter(
pk__in=answer.question.calc_dependents
):
print(f"recalculating {question} from domain logic _create_")
document = models.Document.objects.filter(pk=answer.document_id).prefetch_related(
*utils.build_document_prefetch_statements(
"family", prefetch_options=True
),
).first()
update_or_create_calc_answer(question, document, None)

return answer

Expand All @@ -190,11 +198,18 @@ def update(cls, answer, validated_data, user: Optional[BaseUser] = None):
if answer.question.type == models.Question.TYPE_TABLE:
answer.create_answer_documents(documents)

for question in models.Question.objects.filter(
pk__in=answer.question.calc_dependents
):
print(f"recalculating {question} from domain logic _update_")
update_or_create_calc_answer(question, answer.document)
root_doc = answer.document.family
root_doc = models.Document.objects.filter(pk=answer.document.family_id).prefetch_related(
*utils.build_document_prefetch_statements(prefetch_options=True)
).first()
print("init structure top level")
struc = structure.FieldSet(root_doc, root_doc.form)

for question in models.Question.objects.filter(
pk__in=answer.question.calc_dependents
):
print(f"recalculating {question} from domain logic _update_")
update_or_create_calc_answer(question, root_doc, struc)

answer.refresh_from_db()
return answer
Expand Down Expand Up @@ -292,7 +307,7 @@ def create(
for question in models.Form.get_all_questions(
[(document.family or document).form_id]
).filter(type=models.Question.TYPE_CALCULATED_FLOAT):
update_or_create_calc_answer(question, document)
update_or_create_calc_answer(question, document, None)
return document

@staticmethod
Expand Down
50 changes: 28 additions & 22 deletions caluma/caluma_form/signals.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import itertools
from django.db.models import Prefetch

from django.db.models.signals import (
m2m_changed,
Expand Down Expand Up @@ -122,28 +123,33 @@ def update_calc_from_form_question(sender, instance, created, **kwargs):
update_or_create_calc_answer(instance.question, document)


@receiver(post_save, sender=models.Answer)
@disable_raw
@filter_events(lambda instance: instance.document and instance.question.calc_dependents)
def update_calc_from_answer(sender, instance, **kwargs):
# If there is no document on the answer it means that it's a default
# answer. They shouldn't trigger a recalculation of a calculated field
# even when they are technically listed as a dependency.
# Also skip non-referenced answers.
if instance.document.family.meta.get("_defer_calculation"):
return

if instance.question.type == models.Question.TYPE_TABLE:
print("skipping update calc of table questions in event layer, because we don't have access to the question slug here")
return

print(f"saved answer to {instance.question.pk}, recalculate dependents:")
document = models.Document.objects.filter(pk=instance.document_id).prefetch_related("family__answers", "family__form__questions").first()
for question in models.Question.objects.filter(
pk__in=instance.question.calc_dependents
):
print(f"- {question.pk}")
update_or_create_calc_answer(question, document)
# @receiver(post_save, sender=models.Answer)
# @disable_raw
# @filter_events(lambda instance: instance.document and instance.question.calc_dependents)
# def update_calc_from_answer(sender, instance, **kwargs):
# # If there is no document on the answer it means that it's a default
# # answer. They shouldn't trigger a recalculation of a calculated field
# # even when they are technically listed as a dependency.
# # Also skip non-referenced answers.
# if instance.document.family.meta.get("_defer_calculation"):
# return
#
# if instance.question.type == models.Question.TYPE_TABLE:
# print("skipping update calc of table questions in event layer, because we don't have access to the question slug here")
# return
#
# print(f"saved answer to {instance.question.pk}, recalculate dependents:")
# document = models.Document.objects.filter(pk=instance.document_id).prefetch_related(
# *build_document_prefetch_statements(
# "family", prefetch_options=True
# ),
# ).first()
#
# for question in models.Question.objects.filter(
# pk__in=instance.question.calc_dependents
# ):
# print(f"- {question.pk}")
# update_or_create_calc_answer(question, document)


@receiver(post_save, sender=models.Document)
Expand Down
92 changes: 90 additions & 2 deletions caluma/caluma_form/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,78 @@
from caluma.caluma_form import models, structure
from django.db.models import Prefetch
from caluma.caluma_form.jexl import QuestionJexl
from time import time


def build_document_prefetch_statements(prefix="", prefetch_options=False):
"""Build needed prefetch statements to performantly fetch a document.
This is needed to reduce the query count when almost all the form data
is needed for a given document, e.g. when recalculating calculated
answers.
"""

question_queryset = models.Question.objects.select_related(
"sub_form", "row_form"
).order_by("-formquestion__sort")

if prefetch_options:
question_queryset = question_queryset.prefetch_related(
Prefetch(
"options",
queryset=models.Option.objects.order_by("-questionoption__sort"),
)
)

if prefix:
prefix += "__"

return [
f"{prefix}answers",
f"{prefix}dynamicoption_set",
Prefetch(
f"{prefix}answers__answerdocument_set",
queryset=models.AnswerDocument.objects.select_related(
"document__form", "document__family"
)
.prefetch_related("document__answers", "document__form__questions")
.order_by("-sort"),
),
Prefetch(
# root form -> questions
f"{prefix}form__questions",
queryset=question_queryset.prefetch_related(
Prefetch(
# root form -> row forms -> questions
"row_form__questions",
queryset=question_queryset,
),
Prefetch(
# root form -> sub forms -> questions
"sub_form__questions",
queryset=question_queryset.prefetch_related(
Prefetch(
# root form -> sub forms -> row forms -> questions
"row_form__questions",
queryset=question_queryset,
),
Prefetch(
# root form -> sub forms -> sub forms -> questions
"sub_form__questions",
queryset=question_queryset.prefetch_related(
Prefetch(
# root form -> sub forms -> sub forms -> row forms -> questions
"row_form__questions",
queryset=question_queryset,
),
),
),
),
),
),
),
]



def update_calc_dependents(slug, old_expr, new_expr):
Expand Down Expand Up @@ -28,11 +101,17 @@ def update_calc_dependents(slug, old_expr, new_expr):
question.save()


def update_or_create_calc_answer(question, document):
def update_or_create_calc_answer(question, document, struc):
root_doc = document.family

struc = structure.FieldSet(root_doc, root_doc.form)
if not struc:
print("init structure")
struc = structure.FieldSet(root_doc, root_doc.form)
else:
print("reusing struc")
start = time()
field = struc.get_field(question.slug)
# print(f"get_field: ", time() - start)

# skip if question doesn't exist in this document structure
if field is None:
Expand All @@ -52,6 +131,13 @@ def update_or_create_calc_answer(question, document):
question=question, document=field.document, defaults={"value": value}
)

for _question in models.Question.objects.filter(
pk__in=field.question.calc_dependents
):
print(f"{question.pk} -> {_question.pk}")
update_or_create_calc_answer(_question, document, struc)



def recalculate_answers_from_document(instance):
"""When a table row is added, update dependent questions"""
Expand All @@ -63,3 +149,5 @@ def recalculate_answers_from_document(instance):
[(instance.family or instance).form_id]
).filter(type=models.Question.TYPE_CALCULATED_FLOAT):
update_or_create_calc_answer(question, instance)


0 comments on commit 3f0a010

Please sign in to comment.