Merge pull request #42 from openedx/cag/partition-tables

feat: partition all tables by year and month
openedx · Jan 30, 2024 · 3897b09 · 3897b09
2 parents 04939b5 + 00ff908
commit 3897b09
Show file tree

Hide file tree

Showing 9 changed files with 17 additions and 8 deletions.
diff --git a/models/base/xapi_events_all_parsed.sql b/models/base/xapi_events_all_parsed.sql
@@ -4,6 +4,7 @@
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_id, verb_id, actor_id, emission_time, event_id)',
     order_by='(org, course_id, verb_id, actor_id, emission_time, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
     post_hook='OPTIMIZE TABLE {{ this }} {{ on_cluster() }} FINAL'
   ) }}
 

diff --git a/models/completion/completion_events.sql b/models/completion/completion_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, verb_id)',
-    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)'
+    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/enrollment/enrollment_events.sql b/models/enrollment/enrollment_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key)',
-    order_by='(org, course_key, emission_time, actor_id, enrollment_mode, event_id)'
+    order_by='(org, course_key, emission_time, actor_id, enrollment_mode, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/forum/forum_events.sql b/models/forum/forum_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, verb_id)',
-    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)'
+    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/grading/grading_events.sql b/models/grading/grading_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, verb_id)',
-    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id)'
+    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 

diff --git a/models/navigation/navigation_events.sql b/models/navigation/navigation_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, object_type)',
-    order_by='(org, course_key, object_type, emission_time, actor_id, starting_position, event_id)'
+    order_by='(org, course_key, object_type, emission_time, actor_id, starting_position, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/problems/problem_events.sql b/models/problems/problem_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, verb_id)',
-    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id)'
+    order_by='(org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/video/video_playback_events.sql b/models/video/video_playback_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, verb_id)',
-    order_by='(org, course_key, verb_id, emission_time, actor_id, video_position, event_id)'
+    order_by='(org, course_key, verb_id, emission_time, actor_id, video_position, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT

diff --git a/models/video/video_transcript_events.sql b/models/video/video_transcript_events.sql
@@ -3,7 +3,8 @@
     schema=env_var('ASPECTS_XAPI_DATABASE', 'xapi'),
     engine=get_engine('ReplacingMergeTree()'),
     primary_key='(org, course_key, video_id)',
-    order_by='(org, course_key, video_id, emission_time, actor_id, cc_enabled, event_id)'
+    order_by='(org, course_key, video_id, emission_time, actor_id, cc_enabled, event_id)',
+    partition_by='(toYYYYMM(emission_time))',
   ) }}
 
 SELECT