diff --git a/models/completion/completion_events.sql b/models/completion/completion_events.sql new file mode 100644 index 00000000..3b8edf59 --- /dev/null +++ b/models/completion/completion_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key, verb_id)', + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)' + ) }} + +SELECT + event_id, + CAST(emission_time, 'DateTime') AS emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/cmi5/result/extensions/progress"') AS progress_percent +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE verb_id = 'http://adlnet.gov/expapi/verbs/progressed' diff --git a/models/completion/fact_completions.sql b/models/completion/fact_completions.sql index a0a45a29..f564ce6c 100644 --- a/models/completion/fact_completions.sql +++ b/models/completion/fact_completions.sql @@ -10,9 +10,9 @@ with completions as ( splitByString('/course/', object_id)[-1], splitByString('/xblock/', object_id)[-1] ) as entity_id, - progress_percent/100 as scaled_progress + cast(progress_percent as Float)/100 as scaled_progress from - {{ source('xapi', 'completions_events') }} + {{ ref('completion_events') }} ) select diff --git a/models/enrollment/enrollment_events.sql b/models/enrollment/enrollment_events.sql new file mode 100644 index 00000000..e69b2b09 --- /dev/null +++ b/models/enrollment/enrollment_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key)', + order_by='(org, course_key, emission_time, actor_id, enrollment_mode, event_id)' + ) }} + +SELECT + event_id, + cast(emission_time as DateTime) as emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE verb_id IN ( + 'http://adlnet.gov/expapi/verbs/registered', + 'http://id.tincanapi.com/verb/unregistered' +) diff --git a/models/enrollment/fact_enrollments.sql b/models/enrollment/fact_enrollments.sql index 2cafd611..b1159953 100644 --- a/models/enrollment/fact_enrollments.sql +++ b/models/enrollment/fact_enrollments.sql @@ -7,7 +7,7 @@ with enrollments as ( enrollment_mode, splitByString('/', verb_id)[-1] as enrollment_status from - {{ source('xapi', 'enrollment_events') }} + {{ ref('enrollment_events') }} ) select diff --git a/models/forum/fact_forum_interactions.sql b/models/forum/fact_forum_interactions.sql index 4612ca93..261e5e94 100644 --- a/models/forum/fact_forum_interactions.sql +++ b/models/forum/fact_forum_interactions.sql @@ -9,6 +9,6 @@ select forum.actor_id as actor_id, forum.verb_id as verb_id from - {{ source('xapi', 'forum_events') }} forum + {{ ref('forum_events') }} forum join {{ source('event_sink', 'course_names') }} courses on (forum.course_key = courses.course_key) diff --git a/models/forum/forum_events.sql b/models/forum/forum_events.sql new file mode 100644 index 00000000..4478a850 --- /dev/null +++ b/models/forum/forum_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key, verb_id)', + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)' + ) }} + +SELECT + event_id, + CAST(emission_time, 'DateTime') AS emission_time, + org, + splitByString('/', course_id)[-1] AS course_key, + object_id, + actor_id, + verb_id +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE JSON_VALUE(event_str, '$.object.definition.type') = 'http://id.tincanapi.com/activitytype/discussion' diff --git a/models/grading/fact_grades.sql b/models/grading/fact_grades.sql index 0e31bc9e..65c0a3e8 100644 --- a/models/grading/fact_grades.sql +++ b/models/grading/fact_grades.sql @@ -16,7 +16,7 @@ with grades as ( actor_id, scaled_score from - {{ source('xapi', 'grading_events') }} + {{ ref('grading_events') }} ) select diff --git a/models/grading/grading_events.sql b/models/grading/grading_events.sql new file mode 100644 index 00000000..30bc9684 --- /dev/null +++ b/models/grading/grading_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key, verb_id)', + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id)' + ) }} + + +SELECT + event_id, + CAST(emission_time, 'DateTime') AS emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + JSON_VALUE(event_str, '$.result.score.scaled') AS scaled_score +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE verb_id IN ('http://id.tincanapi.com/verb/earned', 'https://w3id.org/xapi/acrossx/verbs/evaluated') diff --git a/models/problems/fact_problem_responses.sql b/models/problems/fact_problem_responses.sql index 832d71dc..eb6ffaa4 100644 --- a/models/problems/fact_problem_responses.sql +++ b/models/problems/fact_problem_responses.sql @@ -9,7 +9,7 @@ with responses as ( success, attempts from - {{ source('xapi', 'problem_events') }} + {{ ref('problem_events') }} where verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated' ) diff --git a/models/problems/int_problem_hints.sql b/models/problems/int_problem_hints.sql index a193a227..d9e6c058 100644 --- a/models/problems/int_problem_hints.sql +++ b/models/problems/int_problem_hints.sql @@ -11,7 +11,7 @@ with hints as ( else 'N/A' end as help_type from - {{ source('xapi', 'problem_events') }} + {{ ref('problem_events') }} where verb_id = 'http://adlnet.gov/expapi/verbs/asked' ) diff --git a/models/problems/problem_events.sql b/models/problems/problem_events.sql new file mode 100644 index 00000000..378bbf3e --- /dev/null +++ b/models/problems/problem_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key, verb_id)', + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id)' + ) }} + +SELECT + event_id, + CAST(emission_time, 'DateTime') AS emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + replaceAll(replaceAll(JSON_VALUE(event_str, '$.result.response'), '\'', '\\\''), '"', '\'') AS responses, + JSON_VALUE(event_str, '$.result.score.scaled') AS scaled_score, + if(verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', CAST(JSON_VALUE(event_str, '$.result.success'), 'Bool'), false) AS success, + JSON_VALUE(event_str, '$.object.definition.interactionType') AS interaction_type, + if(verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', CAST(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"'), 'Int16'), 0) AS attempts +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE verb_id IN ('https://w3id.org/xapi/acrossx/verbs/evaluated', 'http://adlnet.gov/expapi/verbs/passed', 'http://adlnet.gov/expapi/verbs/asked') diff --git a/models/video/fact_video_plays.sql b/models/video/fact_video_plays.sql index 35d7d430..827e0769 100644 --- a/models/video/fact_video_plays.sql +++ b/models/video/fact_video_plays.sql @@ -8,7 +8,7 @@ with plays as ( splitByString('/xblock/', object_id)[-1] as video_id, actor_id from - {{ source('xapi', 'video_playback_events') }} + {{ ref('video_playback_events') }} where verb_id = 'https://w3id.org/xapi/video/verbs/played' ) diff --git a/models/video/video_playback_events.sql b/models/video/video_playback_events.sql new file mode 100644 index 00000000..1041c528 --- /dev/null +++ b/models/video/video_playback_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized='materialized_view', + engine='MergeTree()', + primary_key='(org, course_key, verb_id)', + order_by='(org, course_key, verb_id, emission_time, actor_id, video_position, event_id)' + ) }} + +SELECT + event_id, + CAST(emission_time, 'DateTime') AS emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + ceil(CAST(coalesce(nullIf(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), nullIf(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), '0.0'), 'Decimal32(2)')) AS video_position +FROM {{ source('xapi', 'xapi_events_all_parsed') }} +WHERE (verb_id IN ('http://adlnet.gov/expapi/verbs/completed', 'http://adlnet.gov/expapi/verbs/initialized', 'http://adlnet.gov/expapi/verbs/terminated', 'https://w3id.org/xapi/video/verbs/paused', 'https://w3id.org/xapi/video/verbs/played', 'https://w3id.org/xapi/video/verbs/seeked')) AND (object_id LIKE '%video+block%')