From 68c4d12e2529baaf544889b8bc1be7d8413f1ca1 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 26 Sep 2024 17:04:55 -0500 Subject: [PATCH 1/2] perf(sql): use last_course_visit mv for learner summary query --- .../queries/fact_learner_summary.sql | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql index bd4af6c02..accf1a97e 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql @@ -1,21 +1,21 @@ with latest_emission_time as ( - select course_key, actor_id, MAX(emission_time) as last_visited - from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation + select org, course_key, actor_id, MAX(emission_time) as last_visited + from {{ ASPECTS_XAPI_DATABASE }}.fact_learner_last_course_visit where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} {% include 'openedx-assets/queries/user_filters.sql' %} - group by course_key, actor_id + group by org, course_key, actor_id ), enrollment_status as ( - select course_key, actor_id, MAX(emission_time) as max_emission_time + select org, course_key, actor_id, MAX(emission_time) as max_emission_time from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} {% include 'openedx-assets/queries/user_filters.sql' %} - group by course_key, actor_id + group by org, course_key, actor_id ), student_status as ( select @@ -58,9 +58,13 @@ select from student_status fss left join enrollment_status fes - on fss.course_key = fes.course_key + on + fss.org = fes.org + and fss.course_key = fes.course_key and fss.actor_id = fes.actor_id left join latest_emission_time let - on fss.course_key = let.course_key + on + fss.org = let.org + and fss.course_key = let.course_key and fss.actor_id = let.actor_id From d7180521a35b8a61e55d347ca9a3d83c9ddb7a07 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 27 Sep 2024 15:47:07 -0500 Subject: [PATCH 2/2] perf(sql): bring int_problem_resuts from dbt to superset fix(sql): flip join direction --- .../queries/fact_learner_summary.sql | 6 +- .../queries/learner_performance.sql | 61 ++++++++++++++++++- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql index accf1a97e..398b2e3b8 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql @@ -58,13 +58,11 @@ select from student_status fss left join enrollment_status fes - on - fss.org = fes.org + on fss.org = fes.org and fss.course_key = fes.course_key and fss.actor_id = fes.actor_id left join latest_emission_time let - on - fss.org = let.org + on fss.org = let.org and fss.course_key = let.course_key and fss.actor_id = let.actor_id diff --git a/tutoraspects/templates/openedx-assets/queries/learner_performance.sql b/tutoraspects/templates/openedx-assets/queries/learner_performance.sql index 6c7829470..83359f531 100644 --- a/tutoraspects/templates/openedx-assets/queries/learner_performance.sql +++ b/tutoraspects/templates/openedx-assets/queries/learner_performance.sql @@ -1,3 +1,62 @@ +-- select one record per (learner, problem, course, org) tuple +-- contains either the first successful attempt +-- or the most recent unsuccessful attempt +-- find the timestamp of the earliest successful response +-- this will be used to pick the xAPI event corresponding to that submission +with + full_responses as ( + select + events.emission_time as emission_time, + events.org as org, + events.course_key as course_key, + events.problem_id as problem_id, + events.object_id as object_id, + events.actor_id as actor_id, + events.responses as responses, + events.success as success, + events.attempts as attempts, + events.interaction_type as interaction_type + from {{ ASPECTS_XAPI_DATABASE }}.responses + join + {{ ASPECTS_XAPI_DATABASE }}.problem_events events using ( + org, course_key, problem_id, actor_id, emission_time + ) + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ), + int_problem_results as ( + select + full_responses.emission_time as emission_time, + full_responses.org as org, + full_responses.course_key as course_key, + blocks.course_name as course_name, + blocks.course_run as course_run, + full_responses.problem_id as problem_id, + blocks.block_name as problem_name, + blocks.display_name_with_location as problem_name_with_location, + blocks.course_order as course_order, + concat( + '', + blocks.display_name_with_location, + '' + ) as problem_link, + full_responses.actor_id as actor_id, + full_responses.responses as responses, + full_responses.success as success, + full_responses.attempts as attempts, + full_responses.interaction_type as interaction_type, + blocks.graded + from full_responses + join + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks blocks + on ( + full_responses.course_key = blocks.course_key + and full_responses.problem_id = blocks.block_id + ) + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ) + select org, course_key, @@ -9,6 +68,6 @@ select course_grade, approving_state from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_student_status -left join {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_results +left join int_problem_results using org, course_key, course_run, actor_id, course_name where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %}