Add included_in_statistics flag to answersession

frontend/src/lib/client/schemas.gen.ts

@@ -542,6 +542,16 @@ export const MilestoneAnswerPublicSchema = {
         answer: {
             type: 'integer',
             title: 'Answer'
+        },
+        included_in_milestone_statistics: {
+            type: 'boolean',
+            title: 'Included In Milestone Statistics',
+            default: false
+        },
+        included_in_milestonegroup_statistics: {
+            type: 'boolean',
+            title: 'Included In Milestonegroup Statistics',
+            default: false
         }
     },
     type: 'object',

frontend/src/lib/client/types.gen.ts

@@ -141,6 +141,8 @@ export type MilestoneAgeScoreCollectionPublic = {
 export type MilestoneAnswerPublic = {
     milestone_id: number;
     answer: number;
+    included_in_milestone_statistics?: boolean;
+    included_in_milestonegroup_statistics?: boolean;
 };
 
 export type MilestoneAnswerSessionPublic = {

mondey_backend/src/mondey_backend/models/milestones.py

@@ -147,6 +147,8 @@ class SubmittedMilestoneImagePublic(SQLModel):
 class MilestoneAnswerPublic(SQLModel):
     milestone_id: int
     answer: int
+    included_in_milestone_statistics: bool = False
+    included_in_milestonegroup_statistics: bool = False
 
 
 class MilestoneAnswer(SQLModel, table=True):
@@ -158,6 +160,8 @@ class MilestoneAnswer(SQLModel, table=True):
     )
     milestone_group_id: int = Field(default=None, foreign_key="milestonegroup.id")
     answer: int
+    included_in_milestone_statistics: bool = False
+    included_in_milestonegroup_statistics: bool = False
 
 
 class MilestoneAnswerSession(SQLModel, table=True):

mondey_backend/src/mondey_backend/routers/statistics.py

@@ -4,7 +4,6 @@
 from collections.abc import Sequence
 
 import numpy as np
-from sqlalchemy import and_
 from sqlmodel import col
 from sqlmodel import select
 
@@ -195,7 +194,6 @@ def calculate_milestone_statistics_by_age(
         MilestoneAgeScoreCollection object which contains a list of MilestoneAgeScore objects,
     one for each month, or None if there are no answers for the milestoneg and no previous statistics.
     """
-    # TODO: when the answersession eventually has an expired flag, this can go again.
     session_expired_days: int = 7
 
     # get the newest statistics for the milestone
@@ -221,6 +219,7 @@ def calculate_milestone_statistics_by_age(
                 col(MilestoneAnswer.answer_session_id) == MilestoneAnswerSession.id,
             )
             .where(MilestoneAnswer.milestone_id == milestone_id)
+            .where(~col(MilestoneAnswer.included_in_milestone_statistics))
             .where(MilestoneAnswerSession.created_at < expiration_date)
         )
     else:
@@ -239,12 +238,8 @@ def calculate_milestone_statistics_by_age(
                 col(MilestoneAnswer.answer_session_id) == MilestoneAnswerSession.id,
             )
             .where(MilestoneAnswer.milestone_id == milestone_id)
-            .where(
-                and_(
-                    col(MilestoneAnswerSession.created_at) > last_statistics.created_at,
-                    col(MilestoneAnswerSession.created_at) <= expiration_date,
-                )  # expired session only which are not in the last statistics
-            )
+            .where(~col(MilestoneAnswer.included_in_milestone_statistics))
+            .where(col(MilestoneAnswerSession.created_at) <= expiration_date)
         )
 
     answers = session.exec(answers_query).all()
@@ -259,6 +254,11 @@ def calculate_milestone_statistics_by_age(
 
         expected_age = _get_expected_age_from_scores(avg_scores)
 
+        for answer in answers:
+            answer.included_in_milestone_statistics = True
+            session.merge(answer)
+        session.commit()
+
         # overwrite last_statistics with updated stuff --> set primary keys explicitly
         return MilestoneAgeScoreCollection(
             milestone_id=milestone_id,
@@ -302,7 +302,6 @@ def calculate_milestonegroup_statistics_by_age(
     one for each month, or None if there are no answers for the milestonegroup and no previous statistics.
     """
 
-    # TODO: when the answersession eventually has an 'expired' flag, this can go again.
     session_expired_days: int = 7
 
     # get the newest statistics for the milestonegroup
@@ -326,9 +325,10 @@ def calculate_milestonegroup_statistics_by_age(
                 col(MilestoneAnswer.answer_session_id) == MilestoneAnswerSession.id,
             )
             .where(MilestoneAnswer.milestone_group_id == milestonegroup_id)
+            .where(~col(MilestoneAnswer.included_in_milestonegroup_statistics))
             .where(
                 MilestoneAnswerSession.created_at
-                < expiration_date  # expired session only
+                <= expiration_date  # expired session only
             )
         )
     else:
@@ -349,15 +349,11 @@ def calculate_milestonegroup_statistics_by_age(
             select(MilestoneAnswer)
             .join(
                 MilestoneAnswerSession,
-                MilestoneAnswer.answer_session_id == MilestoneAnswerSession.id,  # type: ignore
+                col(MilestoneAnswer.answer_session_id) == MilestoneAnswerSession.id,
             )
             .where(MilestoneAnswer.milestone_group_id == milestonegroup_id)
-            .where(
-                and_(
-                    MilestoneAnswerSession.created_at > last_statistics.created_at,  # type: ignore
-                    MilestoneAnswerSession.created_at <= expiration_date,  # type: ignore
-                )
-            )  # expired session only which are not in the last statistics
+            .where(~col(MilestoneAnswer.included_in_milestonegroup_statistics))
+            .where(MilestoneAnswerSession.created_at <= expiration_date)
         )
 
     answers = session.exec(answer_query).all()
@@ -371,6 +367,12 @@ def calculate_milestonegroup_statistics_by_age(
             answers, child_ages, count=count, avg=avg_scores, stddev=stddev_scores
         )
 
+        # update answer.included_in_milestonegroup_statistics to True
+        for answer in answers:
+            answer.included_in_milestonegroup_statistics = True
+            session.merge(answer)
+        session.commit()
+
         return MilestoneGroupAgeScoreCollection(
             milestone_group_id=milestonegroup_id,
             scores=[

mondey_backend/tests/conftest.py

@@ -241,19 +241,27 @@ def session(children: list[dict], monkeypatch: pytest.MonkeyPatch):
             MilestoneAnswerSession(
                 child_id=1,
                 user_id=3,
-                created_at=datetime.datetime(
-                    last_month.year, last_month.month, last_month.day
-                ),
+                created_at=datetime.datetime(last_month.year, last_month.month, 15),
             )
         )
         session.add(
             MilestoneAnswer(
-                answer_session_id=1, milestone_id=1, milestone_group_id=1, answer=1
+                answer_session_id=1,
+                milestone_id=1,
+                milestone_group_id=1,
+                answer=1,
+                included_in_milestone_statistics=True,
+                included_in_milestonegroup_statistics=True,
             )
         )
         session.add(
             MilestoneAnswer(
-                answer_session_id=1, milestone_id=2, milestone_group_id=1, answer=0
+                answer_session_id=1,
+                milestone_id=2,
+                milestone_group_id=1,
+                answer=0,
+                included_in_milestone_statistics=True,
+                included_in_milestonegroup_statistics=True,
             )
         )
         # add another (current) milestone answer session for child 1 / user (id 3) with 2 answers to the same questions
@@ -279,7 +287,10 @@ def session(children: list[dict], monkeypatch: pytest.MonkeyPatch):
         )
         session.add(
             MilestoneAnswer(
-                answer_session_id=3, milestone_id=7, milestone_group_id=2, answer=2
+                answer_session_id=3,
+                milestone_id=7,
+                milestone_group_id=2,
+                answer=2,
             )
         )
         # add a research group (that user with id 3 is part of, and researcher with id 2 has access to)
@@ -462,16 +473,14 @@ def session(children: list[dict], monkeypatch: pytest.MonkeyPatch):
 def statistics_session(session):
     today = datetime.datetime.today()
     last_month = today - relativedelta(months=1)
-    two_weeks_ago = today - relativedelta(weeks=2)
 
     # add another expired milestoneanswersession for milestones 1, 2 for child
+    # this answersession is not part of the statistics yet
     session.add(
         MilestoneAnswerSession(
             child_id=1,
             user_id=3,
-            created_at=datetime.datetime(
-                two_weeks_ago.year, two_weeks_ago.month, two_weeks_ago.day
-            ),
+            created_at=datetime.datetime(today.year, last_month.month, 20),
         )
     )
     session.add(
@@ -508,7 +517,7 @@ def statistics_session(session):
             created_at=datetime.datetime(
                 last_month.year,
                 last_month.month,
-                last_month.day + 2,  # between answersessions -> recompute
+                17,  # between answersessions -> recompute
             ),
         )
     )
@@ -520,7 +529,7 @@ def statistics_session(session):
             created_at=datetime.datetime(
                 last_month.year,
                 last_month.month,
-                last_month.day + 2,  # between answersessions -> recompute
+                17,  # between answersessions -> recompute
             ),
         )
     )
@@ -569,7 +578,7 @@ def sigma(age, lower, upper, value):
             created_at=datetime.datetime(
                 last_month.year,
                 last_month.month,
-                last_month.day + 2,  # between answersessions -> recompute
+                17,  # between answersessions -> recompute
             ),
         )
     )

mondey_backend/tests/routers/test_users.py

@@ -2,6 +2,9 @@
 import pathlib
 
 from fastapi.testclient import TestClient
+from sqlmodel import select
+
+from mondey_backend.models.milestones import MilestoneAnswer
 
 
 def _is_approx_now(iso_date_string: str, delta=datetime.timedelta(hours=1)) -> bool:
@@ -177,8 +180,18 @@ def test_get_milestone_answers_child1_current_answer_session(user_client: TestCl
     assert response.json()["id"] == 2
     assert response.json()["child_id"] == 1
     assert response.json()["answers"] == {
-        "1": {"milestone_id": 1, "answer": 3},
-        "2": {"milestone_id": 2, "answer": 2},
+        "1": {
+            "milestone_id": 1,
+            "answer": 3,
+            "included_in_milestone_statistics": False,
+            "included_in_milestonegroup_statistics": False,
+        },
+        "2": {
+            "milestone_id": 2,
+            "answer": 2,
+            "included_in_milestone_statistics": False,
+            "included_in_milestonegroup_statistics": False,
+        },
     }
     assert _is_approx_now(response.json()["created_at"])
 
@@ -191,7 +204,12 @@ def test_update_milestone_answer_no_current_answer_session(
 
     # child 2 is 20 months old, so milestones 4
     assert current_answer_session["answers"]["4"]["answer"] == -1
-    new_answer = {"milestone_id": 4, "answer": 2}
+    new_answer = {
+        "milestone_id": 4,
+        "answer": 2,
+        "included_in_milestone_statistics": False,
+        "included_in_milestonegroup_statistics": False,
+    }
     response = user_client.put(
         f"/users/milestone-answers/{current_answer_session['id']}", json=new_answer
     )
@@ -203,8 +221,18 @@ def test_update_milestone_answer_no_current_answer_session(
 
 def test_update_milestone_answer_update_existing_answer(user_client: TestClient):
     current_answer_session = user_client.get("/users/milestone-answers/1").json()
-    assert current_answer_session["answers"]["1"] == {"milestone_id": 1, "answer": 3}
-    new_answer = {"milestone_id": 1, "answer": 2}
+    assert current_answer_session["answers"]["1"] == {
+        "milestone_id": 1,
+        "answer": 3,
+        "included_in_milestone_statistics": False,
+        "included_in_milestonegroup_statistics": False,
+    }
+    new_answer = {
+        "milestone_id": 1,
+        "answer": 2,
+        "included_in_milestone_statistics": False,
+        "included_in_milestonegroup_statistics": False,
+    }
     response = user_client.put(
         f"/users/milestone-answers/{current_answer_session['id']}", json=new_answer
     )
@@ -352,7 +380,16 @@ def test_update_current_child_answers_no_prexisting(
     assert response.status_code == 404
 
 
-def test_get_summary_feedback_for_session(user_client: TestClient):
+def test_get_summary_feedback_for_session(user_client: TestClient, session):
+    answers = session.exec(
+        select(MilestoneAnswer).where(MilestoneAnswer.answer_session_id == 1)
+    ).all()
+    for answer in answers:
+        answer.included_in_milestone_statistics = False
+        answer.included_in_milestonegroup_statistics = False
+        session.merge(answer)
+    session.commit()
+
     response = user_client.get("/users/feedback/answersession=1/summary")
     assert response.status_code == 200
     assert response.json() == {"1": 2}
@@ -363,7 +400,15 @@ def test_get_summary_feedback_for_session_invalid(user_client: TestClient):
     assert response.status_code == 404
 
 
-def test_get_detailed_feedback_for_session(user_client: TestClient):
+def test_get_detailed_feedback_for_session(user_client: TestClient, session):
+    answers = session.exec(
+        select(MilestoneAnswer).where(MilestoneAnswer.answer_session_id == 1)
+    ).all()
+    for answer in answers:
+        answer.included_in_milestone_statistics = False
+        answer.included_in_milestonegroup_statistics = False
+        session.merge(answer)
+    session.commit()
     response = user_client.get("/users/feedback/answersession=1/detailed")
     assert response.status_code == 200
     assert response.json() == {"1": {"1": 2, "2": 2}}