From 11c02f89f4a2866cafb2ac2b24c543242e0fd491 Mon Sep 17 00:00:00 2001 From: Sara Burns Date: Wed, 11 Dec 2024 13:36:11 -0500 Subject: [PATCH] testing --- macros/common_filters.sql | 16 +++ models/grading/at_risk_learners.sql | 20 ++++ models/video/fact_video_plays.sql | 1 + models/video/fact_watched_video_segments.sql | 110 +++++++++++++++++++ models/video/watched_video_duration.sql | 15 +-- 5 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 macros/common_filters.sql create mode 100644 models/grading/at_risk_learners.sql create mode 100644 models/video/fact_watched_video_segments.sql diff --git a/macros/common_filters.sql b/macros/common_filters.sql new file mode 100644 index 00000000..e236d77a --- /dev/null +++ b/macros/common_filters.sql @@ -0,0 +1,16 @@ +{% macro common_filters() -%} + and ( + ( + {org_filter:String} <> '[]' + and org in cast({org_filter:String}, 'Array(String)') + ) + or {org_filter:String} = '[]' + ) + and ( + ( + {course_key_filter:String} <> '[]' + and course_key in cast({course_key_filter:String}, 'Array(String)') + ) + or {course_key_filter:String} = '[]' + ) +{%- endmacro %} \ No newline at end of file diff --git a/models/grading/at_risk_learners.sql b/models/grading/at_risk_learners.sql new file mode 100644 index 00000000..99d08765 --- /dev/null +++ b/models/grading/at_risk_learners.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized="materialized_view", + engine=get_engine("ReplacingMergeTree()"), + primary_key="(org, course_key, actor_id)", + order_by="(org, course_key, actor_id)", + ) +}} + +with + page_visits as ( + select org, course_key, actor_id, max(emission_time) as last_visited + from {{ ref('fact_learner_last_course_visit') }} + where emission_time < subtractDays(now(), 7) + group by org, course_key, actor_id + ) +select org, course_key, learners.actor_id as actor_id +from {{ ref('fact_student_status') }} learners +join page_visits using (org, course_key, actor_id) +where approving_state = 'failed' and enrollment_status = 'registered' diff --git a/models/video/fact_video_plays.sql b/models/video/fact_video_plays.sql index 5ab4bd8e..8a38e0d1 100644 --- a/models/video/fact_video_plays.sql +++ b/models/video/fact_video_plays.sql @@ -14,6 +14,7 @@ with actor_id from {{ ref("video_playback_events") }} where verb_id = 'https://w3id.org/xapi/video/verbs/played' + {{ common_filters() }} ) select diff --git a/models/video/fact_watched_video_segments.sql b/models/video/fact_watched_video_segments.sql new file mode 100644 index 00000000..f063d0ae --- /dev/null +++ b/models/video/fact_watched_video_segments.sql @@ -0,0 +1,110 @@ +with + video_events as ( + select + emission_time, + org, + course_key, + splitByString('/xblock/', object_id)[-1] as video_id, + object_id, + actor_id, + verb_id, + video_position, + video_duration + from {{ ref('video_playback_events') }} + where 1 = 1 {{ common_filters() }} + ), + starts as ( + select * + from video_events + where verb_id = 'https://w3id.org/xapi/video/verbs/played' + ), + ends as ( + select * + from video_events + where + verb_id in ( + 'http://adlnet.gov/expapi/verbs/completed', + 'https://w3id.org/xapi/video/verbs/seeked', + 'https://w3id.org/xapi/video/verbs/paused', + 'http://adlnet.gov/expapi/verbs/terminated' + ) + ), + segments as ( + select + starts.org as org, + starts.course_key as course_key, + starts.video_id as video_id, + starts.actor_id, + starts.object_id as object_id, + cast(starts.video_position as Int32) as start_position, + cast(ends.video_position as Int32) as end_position, + starts.emission_time as started_at, + ends.emission_time as ended_at, + ends.verb_id as end_type, + starts.video_duration as video_duration + from starts left + asof join + ends + on ( + starts.org = ends.org + and starts.course_key = ends.course_key + and starts.video_id = ends.video_id + and starts.actor_id = ends.actor_id + and starts.emission_time < ends.emission_time + ) + ), + enriched_segments as ( + select + segments.org as org, + segments.course_key as course_key, + blocks.course_name as course_name, + blocks.course_run as course_run, + blocks.section_with_name as section_with_name, + blocks.subsection_with_name as subsection_with_name, + blocks.block_name as video_name, + blocks.display_name_with_location as video_name_with_location, + segments.actor_id as actor_id, + segments.object_id as object_id, + segments.started_at as started_at, + segments.start_position - (segments.start_position % 5) as start_position, + segments.end_position - (segments.end_position % 5) as end_position, + segments.video_duration as video_duration, + segments.video_id as video_id + from segments + join + {{ ref('dim_course_blocks_extended') }} blocks + on ( + segments.course_key = blocks.course_key + and segments.video_id = blocks.block_id + ) + where 1 = 1 {{ common_filters() }} + ) + +select + org, + course_key, + course_name, + course_run, + section_with_name, + subsection_with_name, + video_name, + video_name_with_location, + video_id, + concat( + '', video_name_with_location, '' + ) as video_link, + actor_id, + started_at, + arrayJoin(range(start_position, end_position, 5)) as segment_start, + video_duration, + CONCAT(toString(segment_start), '-', toString(segment_start + 4)) as segment_range, + start_position, + username, + name, + email +from enriched_segments +left outer join + {{ ref('dim_user_pii') }} users + on (actor_id like 'mailto:%' and SUBSTRING(actor_id, 8) = users.email) + or actor_id = toString(users.external_user_id) +order by segment_start diff --git a/models/video/watched_video_duration.sql b/models/video/watched_video_duration.sql index bcd3bd02..b89063a5 100644 --- a/models/video/watched_video_duration.sql +++ b/models/video/watched_video_duration.sql @@ -21,20 +21,7 @@ with 'http://adlnet.gov/expapi/verbs/terminated', 'https://w3id.org/xapi/video/verbs/seeked' ) - and ( - ( - {org_filter:String} <> '[]' - and org in cast({org_filter:String}, 'Array(String)') - ) - or {org_filter:String} = '[]' - ) - and ( - ( - {course_key_filter:String} <> '[]' - and course_key in cast({course_key_filter:String}, 'Array(String)') - ) - or {course_key_filter:String} = '[]' - ) + {{ common_filters() }} ), matches as ( select