diff --git a/models/base/xapi_events_all_parsed.sql b/models/base/xapi_events_all_parsed.sql index 9947f815..48c9ad06 100644 --- a/models/base/xapi_events_all_parsed.sql +++ b/models/base/xapi_events_all_parsed.sql @@ -4,6 +4,7 @@ engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_id, verb_id, actor_id, emission_time, event_id)', order_by='(org, course_id, verb_id, actor_id, emission_time, event_id)', + partition_by='(toYYYYMM(emission_time))', post_hook='OPTIMIZE TABLE {{ this }} {{ on_cluster() }} FINAL' ) }} diff --git a/models/completion/completion_events.sql b/models/completion/completion_events.sql index 212ed00c..b710f1c5 100644 --- a/models/completion/completion_events.sql +++ b/models/completion/completion_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, verb_id)', - order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)' + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/enrollment/enrollment_events.sql b/models/enrollment/enrollment_events.sql index 4389dcae..f0d25f3b 100644 --- a/models/enrollment/enrollment_events.sql +++ b/models/enrollment/enrollment_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key)', - order_by='(org, course_key, emission_time, actor_id, enrollment_mode, event_id)' + order_by='(org, course_key, emission_time, actor_id, enrollment_mode, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/forum/forum_events.sql b/models/forum/forum_events.sql index 54c2cf29..64fb6b72 100644 --- a/models/forum/forum_events.sql +++ b/models/forum/forum_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, verb_id)', - order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)' + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/grading/grading_events.sql b/models/grading/grading_events.sql index 4b1356d4..15a68e3c 100644 --- a/models/grading/grading_events.sql +++ b/models/grading/grading_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, verb_id)', - order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id)' + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} diff --git a/models/navigation/navigation_events.sql b/models/navigation/navigation_events.sql index 782c43e8..24a25201 100644 --- a/models/navigation/navigation_events.sql +++ b/models/navigation/navigation_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, object_type)', - order_by='(org, course_key, object_type, emission_time, actor_id, starting_position, event_id)' + order_by='(org, course_key, object_type, emission_time, actor_id, starting_position, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/problems/problem_events.sql b/models/problems/problem_events.sql index 798e6bd2..bb59fc25 100644 --- a/models/problems/problem_events.sql +++ b/models/problems/problem_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, verb_id)', - order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id)' + order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/video/video_playback_events.sql b/models/video/video_playback_events.sql index be31991a..d89a819b 100644 --- a/models/video/video_playback_events.sql +++ b/models/video/video_playback_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, verb_id)', - order_by='(org, course_key, verb_id, emission_time, actor_id, video_position, event_id)' + order_by='(org, course_key, verb_id, emission_time, actor_id, video_position, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT diff --git a/models/video/video_transcript_events.sql b/models/video/video_transcript_events.sql index 8f9e5a5f..9798566e 100644 --- a/models/video/video_transcript_events.sql +++ b/models/video/video_transcript_events.sql @@ -3,7 +3,8 @@ schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'), engine=get_engine('ReplacingMergeTree()'), primary_key='(org, course_key, video_id)', - order_by='(org, course_key, video_id, emission_time, actor_id, cc_enabled, event_id)' + order_by='(org, course_key, video_id, emission_time, actor_id, cc_enabled, event_id)', + partition_by='(toYYYYMM(emission_time))', ) }} SELECT