diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index ecbf760f3..140866516 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -387,7 +387,7 @@ # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v2.7"), + ("DBT_BRANCH", "v3.1.1"), # Path to the dbt project inside the repository ("DBT_REPOSITORY_PATH", "aspects-dbt"), # This is a pip compliant list of Python packages to install to run dbt diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0002_raw_xapi_table.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0002_raw_xapi_table.py index 6c558c3a0..5d7d0b802 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0002_raw_xapi_table.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0002_raw_xapi_table.py @@ -30,56 +30,6 @@ def upgrade(): PRIMARY KEY (emission_time, event_id); """ ) - op.execute( - f""" - -- Processed table that Superset reads from - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - {on_cluster} - ( - event_id UUID NOT NULL, - verb_id String NOT NULL, - actor_id String NOT NULL, - object_id String NOT NULL, - org String NOT NULL, - course_id String NOT NULL, - emission_time DateTime64(6) NOT NULL, - event_str String NOT NULL - ) ENGINE {engine} - ORDER BY (org, course_id, verb_id, actor_id, emission_time, event_id) - PRIMARY KEY (org, course_id, verb_id, actor_id, emission_time, event_id); - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the raw table to processed table - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - AS - SELECT - event_id as event_id, - JSON_VALUE(event_str, '$.verb.id') as verb_id, - COALESCE( - NULLIF(JSON_VALUE(event_str, '$.actor.account.name'), ''), - NULLIF(JSON_VALUE(event_str, '$.actor.mbox'), ''), - JSON_VALUE(event_str, '$.actor.mbox_sha1sum') - ) as actor_id, - JSON_VALUE(event_str, '$.object.id') as object_id, - -- If the contextActivities parent is a course, use that. Otherwise use the object id for the course id - if( - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].definition.type') - = 'http://adlnet.gov/expapi/activities/course', - JSON_VALUE(event_str, '$.context.contextActivities.parent[0].id'), - JSON_VALUE(event_str, '$.object.id') - ) as course_id, - get_org_from_course_url(course_id) as org, - emission_time as emission_time, - event_str as event_str - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}; - """ - ) def downgrade(): @@ -87,11 +37,3 @@ def downgrade(): "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}" f"{on_cluster};" ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }}" - f"{on_cluster};" - ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0003_enrollment.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0003_enrollment.py deleted file mode 100644 index 9a863f6e4..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0003_enrollment.py +++ /dev/null @@ -1,71 +0,0 @@ -from alembic import op -import sqlalchemy as sa - -revision = "0003" -down_revision = "0002" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" - - -def upgrade(): - op.execute( - """ - SET allow_experimental_object_type=1; - """ - ) - op.execute( - f""" - -- MV target table for enrollment xAPI events - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `enrollment_mode` LowCardinality(String) - ) ENGINE = {engine} - PRIMARY KEY (org, course_id) - ORDER BY (org, course_id, actor_id, enrollment_mode, emission_time); - """ - ) - op.execute( - f""" - -- Processed table that Superset reads from - -- Materialized view that moves data from the processed xAPI table to - -- the enrollment events table - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} AS - SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/registered', - 'http://id.tincanapi.com/verb/unregistered' - ); - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0004_video.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0004_video.py deleted file mode 100644 index 96b2c4d45..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0004_video.py +++ /dev/null @@ -1,78 +0,0 @@ -from alembic import op -import sqlalchemy as sa - -revision = "0004" -down_revision = "0003" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" - - -def upgrade(): - op.execute( - """ - SET allow_experimental_object_type=1; - """ - ) - op.execute( - f""" - -- MV target table for video playback xAPI events - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `video_position` Float32 NOT NULL - ) ENGINE = {engine} - PRIMARY KEY (org, course_id, verb_id) - ORDER BY (org, course_id, verb_id, actor_id); - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the processed xAPI table to - -- the video playback events table - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} AS - SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - cast(coalesce( - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), - '0.0' - ) as Float32) as video_position - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/completed', - 'http://adlnet.gov/expapi/verbs/initialized', - 'http://adlnet.gov/expapi/verbs/terminated', - 'https://w3id.org/xapi/video/verbs/paused', - 'https://w3id.org/xapi/video/verbs/played', - 'https://w3id.org/xapi/video/verbs/seeked' - ); - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0005_problem.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0005_problem.py deleted file mode 100644 index 977660722..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0005_problem.py +++ /dev/null @@ -1,93 +0,0 @@ -from alembic import op -import sqlalchemy as sa - -revision = "0005" -down_revision = "0004" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" - - -def upgrade(): - op.execute( - """ - SET allow_experimental_object_type=1; - """ - ) - op.execute( - f""" - -- MV target table for problem interaction xAPI events - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 - ) ENGINE = {engine} - PRIMARY KEY (org, course_id, verb_id) - ORDER BY (org, course_id, verb_id, actor_id); - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the processed xAPI table to - -- the problem events table - -- n.b. this query omits browser problem_checked events, as they do not - -- contain any information that the server events don't have and including - -- them would heavily skew the distribution of values in the problem - -- response fields (responses, scaled_score, etc) - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} AS - SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts - FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0006_navigation.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0006_navigation.py deleted file mode 100644 index f885643c7..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0006_navigation.py +++ /dev/null @@ -1,83 +0,0 @@ -from alembic import op -import sqlalchemy as sa - -revision = "0006" -down_revision = "0005" -branch_labels = None -depends_on = None -on_cluster = "ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}'" if ("{{CLICKHOUSE_CLUSTER_NAME}}") else "" -engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" - - -def upgrade(): - op.execute( - f""" - -- MV target table for navigation xAPI events - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `object_type` LowCardinality(String) NOT NULL, - `starting_position` Int16, - `ending_point` String - ) ENGINE = {engine} - PRIMARY KEY (org, course_id, object_type) - ORDER BY (org, course_id, object_type, actor_id); - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the processed xAPI table to - -- the enrollment events table - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} AS - SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.type') AS object_type, - -- clicking a link and selecting a module outline have no starting-position field - if ( - object_type in ( - 'http://adlnet.gov/expapi/activities/link', - 'http://adlnet.gov/expapi/activities/module' - ), - 0, - cast(JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/starting-position"' - ) as Int16) - ) AS starting_position, - JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/ending-point"' - ) AS ending_point - FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE verb_id IN ( - 'https://w3id.org/xapi/dod-isd/verbs/navigated' - ); - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0007_event_sink.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0007_event_sink.py index 2cbc8faa2..b8ed02c6a 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0007_event_sink.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0007_event_sink.py @@ -2,7 +2,7 @@ import sqlalchemy as sa revision = "0007" -down_revision = "0006" +down_revision = "0002" branch_labels = None depends_on = None on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0009_prefer_course_key.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0009_prefer_course_key.py deleted file mode 100644 index 0e516d78b..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0009_prefer_course_key.py +++ /dev/null @@ -1,455 +0,0 @@ -from dataclasses import dataclass - -from alembic import op - -revision = "0009" -down_revision = "0008" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -@dataclass -class MvMigration: - table_name: str - old_ddl: str - new_ddl: str - mv_name: str - old_mv_query: str - new_mv_query: str - - -OLD_ENROLLMENT_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `enrollment_mode` LowCardinality(String) -) ENGINE = {engine} -PRIMARY KEY (org, course_id) -ORDER BY (org, course_id, actor_id, enrollment_mode, emission_time); -""" - -NEW_ENROLLMENT_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `enrollment_mode` LowCardinality(String) -) ENGINE = {engine} -PRIMARY KEY (org, course_key) -ORDER BY (org, course_key, actor_id, enrollment_mode, emission_time); -""" - -OLD_ENROLLMENT_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/registered', - 'http://id.tincanapi.com/verb/unregistered' -); -""" - -NEW_ENROLLMENT_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/registered', - 'http://id.tincanapi.com/verb/unregistered' -); -""" - -OLD_VIDEO_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `video_position` Float32 NOT NULL -) ENGINE = {engine} -PRIMARY KEY (org, course_id, verb_id) -ORDER BY (org, course_id, verb_id, actor_id); -""" - -NEW_VIDEO_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `video_position` Float32 NOT NULL -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, actor_id); -""" - -OLD_VIDEO_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - cast(coalesce( - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), - '0.0' - ) as Float32) as video_position -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/completed', - 'http://adlnet.gov/expapi/verbs/initialized', - 'http://adlnet.gov/expapi/verbs/terminated', - 'https://w3id.org/xapi/video/verbs/paused', - 'https://w3id.org/xapi/video/verbs/played', - 'https://w3id.org/xapi/video/verbs/seeked' -); -""" - -NEW_VIDEO_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - cast(coalesce( - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), - '0.0' - ) as Float32) as video_position -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/completed', - 'http://adlnet.gov/expapi/verbs/initialized', - 'http://adlnet.gov/expapi/verbs/terminated', - 'https://w3id.org/xapi/video/verbs/paused', - 'https://w3id.org/xapi/video/verbs/played', - 'https://w3id.org/xapi/video/verbs/seeked' -); -""" - -OLD_PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_id, verb_id) -ORDER BY (org, course_id, verb_id, actor_id); -""" - -NEW_PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, actor_id); -""" - -OLD_PROBLEM_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - -NEW_PROBLEM_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - -OLD_NAVIGATION_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_id` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `object_type` LowCardinality(String) NOT NULL, - `starting_position` Int16, - `ending_point` String -) ENGINE = {engine} -PRIMARY KEY (org, course_id, object_type) -ORDER BY (org, course_id, object_type, actor_id); -""" - -NEW_NAVIGATION_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `object_type` LowCardinality(String) NOT NULL, - `starting_position` Int16, - `ending_point` String -) ENGINE = {engine} -PRIMARY KEY (org, course_key, object_type) -ORDER BY (org, course_key, object_type, actor_id); -""" - -OLD_NAVIGATION_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - course_id, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.type') AS object_type, - -- clicking a link and selecting a module outline have no starting-position field - if ( - object_type in ( - 'http://adlnet.gov/expapi/activities/link', - 'http://adlnet.gov/expapi/activities/module' - ), - 0, - cast(JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/starting-position"' - ) as Int16) - ) AS starting_position, - JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/ending-point"' - ) AS ending_point -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'https://w3id.org/xapi/dod-isd/verbs/navigated' -); -""" - -NEW_NAVIGATION_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.type') AS object_type, - -- clicking a link and selecting a module outline have no starting-position field - if ( - object_type in ( - 'http://adlnet.gov/expapi/activities/link', - 'http://adlnet.gov/expapi/activities/module' - ), - 0, - cast(JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/starting-position"' - ) as Int16) - ) AS starting_position, - JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/ending-point"' - ) AS ending_point -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'https://w3id.org/xapi/dod-isd/verbs/navigated' -); -""" - -MIGRATIONS = [ - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }}", - OLD_ENROLLMENT_DDL, - NEW_ENROLLMENT_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_TRANSFORM_MV }}", - OLD_ENROLLMENT_QUERY, - NEW_ENROLLMENT_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }}", - OLD_VIDEO_DDL, - NEW_VIDEO_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_TRANSFORM_MV }}", - OLD_VIDEO_QUERY, - NEW_VIDEO_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}", - OLD_PROBLEM_DDL, - NEW_PROBLEM_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}", - OLD_PROBLEM_QUERY, - NEW_PROBLEM_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }}", - OLD_NAVIGATION_DDL, - NEW_NAVIGATION_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_TRANSFORM_MV }}", - OLD_NAVIGATION_QUERY, - NEW_NAVIGATION_QUERY, - ), -] - - -def migrate(table_name, mv_name, table_ddl, mv_query): - # for each table and materialized view to migrate: - # - drop the existing table and materialized view if they exist - # - create the new table - # - load data into the new table using the new MV query - # - create the materialized view using the new query - op.execute(f"DROP TABLE IF EXISTS {table_name} {on_cluster}") - op.execute(f"DROP VIEW IF EXISTS {mv_name} {on_cluster}") - op.execute(table_ddl) - op.execute(f"INSERT INTO {table_name} {mv_query}") - mv_ddl = ( - f"CREATE MATERIALIZED VIEW IF NOT EXISTS {mv_name} {on_cluster} " - f"TO {table_name} AS {mv_query}" - ) - op.execute(mv_ddl) - - -def upgrade(): - for migration in MIGRATIONS: - migrate( - migration.table_name, - migration.mv_name, - migration.new_ddl, - migration.new_mv_query, - ) - - -def downgrade(): - for migration in MIGRATIONS: - migrate( - migration.table_name, - migration.mv_name, - migration.old_ddl, - migration.old_mv_query, - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py index e1cc9854d..35cf13bee 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0010_course_dictionaries.py @@ -2,7 +2,7 @@ import sqlalchemy as sa revision = "0010" -down_revision = "0009" +down_revision = "0008" branch_labels = None depends_on = None on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0012_vector_replacingmergetree_xapi_parsed.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0012_vector_replacingmergetree_xapi_parsed.py deleted file mode 100644 index 0e65b1c2a..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0012_vector_replacingmergetree_xapi_parsed.py +++ /dev/null @@ -1,199 +0,0 @@ -from alembic import op - -revision = "0012" -down_revision = "0011" -branch_labels = None -depends_on = None - -DESTINATION_TABLE = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }}" -TMP_TABLE_NEW = f"{DESTINATION_TABLE}_tmp_{revision}" -TMP_TABLE_ORIG = f"{DESTINATION_TABLE}_tmp_mergetree_{revision}" -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -old_engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -def upgrade(): - # 1. Create our new temp table with the desired changes - op.execute( - f""" - CREATE OR REPLACE TABLE {TMP_TABLE_NEW} - {on_cluster} - ( - event_id UUID, - verb_id String, - actor_id String, - object_id String, - org String, - course_id String, - emission_time DateTime, - event_str String - ) - ENGINE = {engine} - PRIMARY KEY (org, course_id, emission_time, verb_id, actor_id, event_id) - ORDER BY (org, course_id, emission_time, verb_id, actor_id, event_id); - """ - ) - # 2. Swap both tables in a single rename statement. New data will flow into - # the new table now and cascade through the MVs and downstream tables per normal. - op.execute(f"RENAME TABLE {DESTINATION_TABLE} TO {TMP_TABLE_ORIG} {on_cluster}") - op.execute(f"RENAME TABLE {TMP_TABLE_NEW} TO {DESTINATION_TABLE} {on_cluster}") - - # 3. Copy in all existing rows from the parent raw table. This will cascade through - # the system and duplicate rows downstream, but the alternative is to potentially - # lose rows in this table while performing a copy. Downstream tables at this - # point are all ReplacingMergeTree, we will force them all do dedupe at the end. - # - # This is the SQL from the current version of the materialized view that - # populates this table. - op.execute( - f""" - INSERT INTO {DESTINATION_TABLE} - (event_id, verb_id, actor_id, object_id, course_id, org, emission_time, event_str) - SELECT - event_id as event_id, - JSON_VALUE(event_str, '$.verb.id') as verb_id, - COALESCE( - NULLIF(JSON_VALUE(event_str, '$.actor.account.name'), ''), - NULLIF(JSON_VALUE(event_str, '$.actor.mbox'), ''), - JSON_VALUE(event_str, '$.actor.mbox_sha1sum') - ) as actor_id, - JSON_VALUE(event_str, '$.object.id') as object_id, - if( - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].definition.type') - = 'http://adlnet.gov/expapi/activities/course', - JSON_VALUE(event_str, '$.context.contextActivities.parent[0].id'), - JSON_VALUE(event_str, '$.object.id') - ) as course_id, - get_org_from_course_url(course_id) as org, - emission_time as emission_time, - event_str as event_str - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}; - """ - ) - # 5. Force deduplication of the existing data. This may take a very long time - # on a larger dataset, but since Aspects isn't in production anywhere yet this - # seems like a reasonable thing to do. If you're looking at this as fodder for - # a future migration, make sure to understand the potential issues here. - optimize() - - # 6. Drop the renamed version of the original table. - op.execute( - f""" - DROP TABLE {TMP_TABLE_ORIG} {on_cluster} - """ - ) - - -def downgrade(): - # 1. Create a new table with the old engine - op.execute( - f""" - CREATE OR REPLACE TABLE {TMP_TABLE_ORIG} - {on_cluster} - ( - event_id UUID, - verb_id String, - actor_id String, - object_id String, - org String, - course_id String, - emission_time DateTime64(6), - event_str String - ) - ENGINE = {old_engine} - PRIMARY KEY (org, course_id, verb_id, actor_id, emission_time, event_id) - ORDER BY (org, course_id, verb_id, actor_id, emission_time, event_id); - """ - ) - # 2. Swap both tables. We can't do this in a single statement because CH Cloud - # uses replicated tables and will error. New data will flow into the new table - # now and cascade through the MVs and downstream tables per normal. - op.execute(f"RENAME TABLE {DESTINATION_TABLE} TO {TMP_TABLE_NEW} {on_cluster}") - op.execute(f"RENAME TABLE {TMP_TABLE_ORIG} TO {DESTINATION_TABLE} {on_cluster}") - - # 3. Copy in all existing rows from the parent raw table. This will cascade through - # the system and duplicate rows downstream, but the alternative is to potentially - # lose rows in this table while performing a copy. Downstream tables at this - # point are all ReplacingMergeTree, we will force them all do dedupe at the end. - # - # This is the SQL from the current version of the materialized view that - # populates this table. - op.execute( - f""" - INSERT INTO {DESTINATION_TABLE} - (event_id, verb_id, actor_id, object_id, course_id, org, emission_time, event_str) - SELECT - event_id as event_id, - JSON_VALUE(event_str, '$.verb.id') as verb_id, - COALESCE( - NULLIF(JSON_VALUE(event_str, '$.actor.account.name'), ''), - NULLIF(JSON_VALUE(event_str, '$.actor.mbox'), ''), - JSON_VALUE(event_str, '$.actor.mbox_sha1sum') - ) as actor_id, - JSON_VALUE(event_str, '$.object.id') as object_id, - if( - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].definition.type') - = 'http://adlnet.gov/expapi/activities/course', - JSON_VALUE(event_str, '$.context.contextActivities.parent[0].id'), - JSON_VALUE(event_str, '$.object.id') - ) as course_id, - get_org_from_course_url(course_id) as org, - emission_time as emission_time, - event_str as event_str - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}; - """ - ) - # 5. Force deduplication of the existing data. This may take a very long time - # on a larger dataset, but since Aspects isn't in production anywhere yet this - # seems like a reasonable thing to do. If you're looking at this as fodder for - # a future migration, make sure to understand the potential issues here. - optimize() - - # 6. Drop the renamed version of the original table. - op.execute( - f""" - DROP TABLE {TMP_TABLE_NEW} {on_cluster} - """ - ) - - -def optimize(): - op.execute( - f""" - OPTIMIZE TABLE {DESTINATION_TABLE} {on_cluster} FINAL - """ - ) - op.execute( - f""" - OPTIMIZE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} - {on_cluster} - FINAL - """ - ) - op.execute( - f""" - OPTIMIZE TABLE - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} - {on_cluster} - FINAL - """ - ) - op.execute( - f""" - OPTIMIZE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} - {on_cluster} - FINAL - """ - ) - op.execute( - f""" - OPTIMIZE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} - {on_cluster} - FINAL - """ - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0013_reorder_mvs.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0013_reorder_mvs.py deleted file mode 100644 index 0d3cd5ca1..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0013_reorder_mvs.py +++ /dev/null @@ -1,478 +0,0 @@ -""" -Updates: -1. The ORDER BY clauses for each top-level materialized view include more relevant - fields. Since these tables now use the ReplacingMergeTree engine, the ORDER BY - fields are used to determine whether a row needs to be updated. -2. The video playback events MV now uses an integer for the video position -3. The emission_time fields now use the DateTime type, as we do not need sub-second - granularity and so can save space by using a smaller data type. -""" -from dataclasses import dataclass - -from alembic import op - -revision = "0013" -down_revision = "0012" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -@dataclass -class MvMigration: - table_name: str - old_ddl: str - new_ddl: str - mv_name: str - old_mv_query: str - new_mv_query: str - - -OLD_ENROLLMENT_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `enrollment_mode` LowCardinality(String) -) ENGINE = {engine} -PRIMARY KEY (org, course_key) -ORDER BY (org, course_key, actor_id, enrollment_mode, emission_time); -""" - - -NEW_ENROLLMENT_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `enrollment_mode` LowCardinality(String) -) ENGINE = {engine} -PRIMARY KEY (org, course_key) -ORDER BY (org, course_key, emission_time, actor_id, enrollment_mode, event_id); -""" - - -OLD_ENROLLMENT_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/registered', - 'http://id.tincanapi.com/verb/unregistered' -); -""" - - -NEW_ENROLLMENT_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.extensions."https://w3id.org/xapi/acrossx/extensions/type"') AS enrollment_mode -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/registered', - 'http://id.tincanapi.com/verb/unregistered' -); -""" - - -OLD_VIDEO_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `video_position` Float32 NOT NULL -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, actor_id); -""" - - -NEW_VIDEO_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `video_position` UInt32 NOT NULL -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, emission_time, actor_id, video_position, event_id); -""" - - -OLD_VIDEO_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - cast(coalesce( - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), - '0.0' - ) as Float32) as video_position -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/completed', - 'http://adlnet.gov/expapi/verbs/initialized', - 'http://adlnet.gov/expapi/verbs/terminated', - 'https://w3id.org/xapi/video/verbs/paused', - 'https://w3id.org/xapi/video/verbs/played', - 'https://w3id.org/xapi/video/verbs/seeked' -); -""" - -NEW_VIDEO_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - ceil(cast(coalesce( - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time"'), ''), - nullif(JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/video/extensions/time-from"'), ''), - '0.0' - ) as Decimal32(2))) as video_position -FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'http://adlnet.gov/expapi/verbs/completed', - 'http://adlnet.gov/expapi/verbs/initialized', - 'http://adlnet.gov/expapi/verbs/terminated', - 'https://w3id.org/xapi/video/verbs/paused', - 'https://w3id.org/xapi/video/verbs/played', - 'https://w3id.org/xapi/video/verbs/seeked' -); -""" - - -OLD_PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, actor_id); -""" - - -NEW_PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id); -""" - - -OLD_PROBLEM_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -NEW_PROBLEM_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -OLD_NAVIGATION_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime64(6) NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `object_type` LowCardinality(String) NOT NULL, - `starting_position` Int16, - `ending_point` String -) ENGINE = {engine} -PRIMARY KEY (org, course_key, object_type) -ORDER BY (org, course_key, object_type, actor_id); -""" - - -NEW_NAVIGATION_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `object_type` LowCardinality(String) NOT NULL, - `starting_position` Int16, - `ending_point` String -) ENGINE = {engine} -PRIMARY KEY (org, course_key, object_type) -ORDER BY (org, course_key, object_type, emission_time, actor_id, starting_position, event_id); -""" - - -OLD_NAVIGATION_QUERY = """ -SELECT - event_id, - emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.type') AS object_type, - -- clicking a link and selecting a module outline have no starting-position field - if ( - object_type in ( - 'http://adlnet.gov/expapi/activities/link', - 'http://adlnet.gov/expapi/activities/module' - ), - 0, - cast(JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/starting-position"' - ) as Int16) - ) AS starting_position, - JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/ending-point"' - ) AS ending_point -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'https://w3id.org/xapi/dod-isd/verbs/navigated' -); -""" - - -NEW_NAVIGATION_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.object.definition.type') AS object_type, - -- clicking a link and selecting a module outline have no starting-position field - if ( - object_type in ( - 'http://adlnet.gov/expapi/activities/link', - 'http://adlnet.gov/expapi/activities/module' - ), - 0, - cast(JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/starting-position"' - ) as Int16) - ) AS starting_position, - JSON_VALUE( - event_str, - '$.context.extensions."http://id.tincanapi.com/extension/ending-point"' - ) AS ending_point -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE verb_id IN ( - 'https://w3id.org/xapi/dod-isd/verbs/navigated' -); -""" - -MIGRATIONS = [ - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_EVENTS_TABLE }}", - OLD_ENROLLMENT_DDL, - NEW_ENROLLMENT_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_ENROLLMENT_TRANSFORM_MV }}", - OLD_ENROLLMENT_QUERY, - NEW_ENROLLMENT_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_EVENTS_TABLE }}", - OLD_VIDEO_DDL, - NEW_VIDEO_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_VIDEO_PLAYBACK_TRANSFORM_MV }}", - OLD_VIDEO_QUERY, - NEW_VIDEO_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}", - OLD_PROBLEM_DDL, - NEW_PROBLEM_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}", - OLD_PROBLEM_QUERY, - NEW_PROBLEM_QUERY, - ), - MvMigration( - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_EVENTS_TABLE }}", - OLD_NAVIGATION_DDL, - NEW_NAVIGATION_DDL, - "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_NAVIGATION_TRANSFORM_MV }}", - OLD_NAVIGATION_QUERY, - NEW_NAVIGATION_QUERY, - ), -] - - -def migrate(table_name, mv_name, table_ddl, mv_query): - # for each table and materialized view to migrate: - # - drop the existing table and materialized view if they exist - # - create the new table - # - load data into the new table using the new MV query - # - create the materialized view using the new query - op.execute(f"DROP TABLE IF EXISTS {table_name} {on_cluster}") - op.execute(f"DROP VIEW IF EXISTS {mv_name} {on_cluster}") - op.execute(table_ddl) - op.execute(f"INSERT INTO {table_name} {mv_query}") - mv_ddl = ( - f"CREATE MATERIALIZED VIEW IF NOT EXISTS {mv_name} {on_cluster} " - f"TO {table_name} AS {mv_query}" - ) - op.execute(mv_ddl) - - -def upgrade(): - for migration in MIGRATIONS: - migrate( - migration.table_name, - migration.mv_name, - migration.new_ddl, - migration.new_mv_query, - ) - - -def downgrade(): - for migration in MIGRATIONS: - migrate( - migration.table_name, - migration.mv_name, - migration.old_ddl, - migration.old_mv_query, - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py index 082882486..69e4dc773 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0014_add_course_names_fields.py @@ -5,7 +5,7 @@ revision = "0014" -down_revision = "0013" +down_revision = "0011" branch_labels = None depends_on = None on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0018_grading_events_mv.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0018_grading_events_mv.py deleted file mode 100644 index b3acd8117..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0018_grading_events_mv.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -create a top-level materialized view for grading events -""" -from alembic import op - - -revision = "0018" -down_revision = "0017" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -def upgrade(): - op.execute( - f""" - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_GRADING_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64 NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `scaled_score` String - ) ENGINE = {engine} - PRIMARY KEY (org, course_key, verb_id) - ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, scaled_score, event_id); - """ - ) - - op.execute( - f""" - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_GRADING_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_GRADING_EVENTS_TABLE }} AS - SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score - FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE - verb_id in ( - 'http://id.tincanapi.com/verb/earned', - 'https://w3id.org/xapi/acrossx/verbs/evaluated' - ); - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_GRADING_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP VIEW IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_GRADING_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0019_forum_events_mv.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0019_forum_events_mv.py deleted file mode 100644 index f6030e3b3..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0019_forum_events_mv.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -create a top-level materialized view for forum events -""" -from alembic import op - - -revision = "0019" -down_revision = "0018" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -def upgrade(): - op.execute( - f""" - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_FORUM_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64 NOT NULL, - `org` String NOT NULL, - `course_key` String NOT NULL, - `object_id` String NOT NULL, - `actor_id` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL - ) ENGINE = {engine} - PRIMARY KEY (org, course_key, verb_id) - ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, event_id); - """ - ) - - op.execute( - f""" - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_FORUM_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_FORUM_EVENTS_TABLE }} AS - SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - org, - splitByString('/', course_id)[-1] AS course_key, - object_id, - actor_id, - verb_id - FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE - JSON_VALUE(event_str, '$.object.definition.type') = 'http://id.tincanapi.com/activitytype/discussion' - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_FORUM_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP VIEW IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_FORUM_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0020_get_org_from_course_url.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0020_get_org_from_course_url.py index 42df3a141..8036b1722 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0020_get_org_from_course_url.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0020_get_org_from_course_url.py @@ -3,7 +3,7 @@ import sqlalchemy as sa revision = "0020" -down_revision = "0019" +down_revision = "0017" branch_labels = None depends_on = None on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0021_completion_events.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0021_completion_events.py deleted file mode 100644 index ceb6e8b81..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0021_completion_events.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -create a top-level materialized view for completion events -""" -from alembic import op - - -revision = "0021" -down_revision = "0020" -branch_labels = None -depends_on = None -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -def upgrade(): - op.execute( - f""" - CREATE TABLE IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_COMPLETION_EVENTS_TABLE }} - {on_cluster} - ( - `event_id` UUID NOT NULL, - `emission_time` DateTime64 NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `progress_percent` Int32, - ) ENGINE = {engine} - PRIMARY KEY (org, course_key, verb_id) - ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, event_id); - """ - ) - - op.execute( - f""" - CREATE MATERIALIZED VIEW IF NOT EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_COMPLETION_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_COMPLETION_EVENTS_TABLE }} AS - SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.extensions."https://w3id.org/xapi/cmi5/result/extensions/progress"') as progress_percent - FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - WHERE - verb_id = 'http://adlnet.gov/expapi/verbs/progressed'; - """ - ) - - -def downgrade(): - op.execute( - "DROP TABLE IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_COMPLETION_EVENTS_TABLE }}" - f"{on_cluster}" - ) - op.execute( - "DROP VIEW IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_COMPLETION_TRANSFORM_MV }}" - f"{on_cluster}" - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0022_new_course_id_loc.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0022_new_course_id_loc.py deleted file mode 100644 index b958b3c62..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0022_new_course_id_loc.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Update xapi_events_all_parsed_mv to parse the course id from -new places due to changes in how multi-question problem_checks -are handled. -""" -from alembic import op - -revision = "0022" -down_revision = "0021" -branch_labels = None -depends_on = None - -DESTINATION_TABLE = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }}" -TMP_TABLE_NEW = f"{DESTINATION_TABLE}_tmp_{revision}" -TMP_TABLE_ORIG = f"{DESTINATION_TABLE}_tmp_mergetree_{revision}" -on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" -old_engine = "ReplicatedMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "MergeTree" -engine = "ReplicatedReplacingMergeTree" if "{{CLICKHOUSE_CLUSTER_NAME}}" else "ReplacingMergeTree" - - -def upgrade(): - op.execute( - """ - SET allow_experimental_object_type=1; - """ - ) - op.execute( - # There is not currently a "CREATE OR REPLACE MATERIALIZED VIEW..." - f""" - DROP VIEW IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }} - {on_cluster}; - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the raw table to processed table - CREATE MATERIALIZED VIEW {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - AS - SELECT - event_id as event_id, - JSON_VALUE(event_str, '$.verb.id') as verb_id, - COALESCE( - NULLIF(JSON_VALUE(event_str, '$.actor.account.name'), ''), - NULLIF(JSON_VALUE(event_str, '$.actor.mbox'), ''), - JSON_VALUE(event_str, '$.actor.mbox_sha1sum') - ) as actor_id, - JSON_VALUE(event_str, '$.object.id') as object_id, - -- If the contextActivities parent is a course, use that. It can be a "course" - -- type, or a "cmi.interaction" type for multiple question problem submissions. - -- Otherwise use the object id for the course id. - multiIf( - -- If the contextActivities parent is a course, use that - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].definition.type' - ) = 'http://adlnet.gov/expapi/activities/course', - JSON_VALUE(event_str, '$.context.contextActivities.parent[0].id'), - -- Else if the contextActivities parent is a GroupActivity, it's a multi - -- question problem and we use the grouping id - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].objectType' - ) in ('Activity', 'GroupActivity'), - JSON_VALUE(event_str, '$.context.contextActivities.grouping[0].id'), - -- Otherwise use the object id - JSON_VALUE(event_str, '$.object.id') - ) as course_id, - get_org_from_course_url(course_id) as org, - emission_time as emission_time, - event_str as event_str - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}; - """ -) - - -def downgrade(): - op.execute( - """ - SET allow_experimental_object_type=1; - """ - ) - op.execute( - # There is not currently a "CREATE OR REPLACE MATERIALIZED VIEW..." - f""" - DROP VIEW IF EXISTS {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }} - {on_cluster}; - """ - ) - op.execute( - f""" - -- Materialized view that moves data from the raw table to processed table - CREATE MATERIALIZED VIEW {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TRANSFORM_MV }} - {on_cluster} - TO {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} - AS - SELECT - event_id as event_id, - JSON_VALUE(event_str, '$.verb.id') as verb_id, - COALESCE( - NULLIF(JSON_VALUE(event_str, '$.actor.account.name'), ''), - NULLIF(JSON_VALUE(event_str, '$.actor.mbox'), ''), - JSON_VALUE(event_str, '$.actor.mbox_sha1sum') - ) as actor_id, - JSON_VALUE(event_str, '$.object.id') as object_id, - -- If the contextActivities parent is a course, use that. Otherwise use the object id for the course id - if( - JSON_VALUE( - event_str, - '$.context.contextActivities.parent[0].definition.type') - = 'http://adlnet.gov/expapi/activities/course', - JSON_VALUE(event_str, '$.context.contextActivities.parent[0].id'), - JSON_VALUE(event_str, '$.object.id') - ) as course_id, - get_org_from_course_url(course_id) as org, - emission_time as emission_time, - event_str as event_str - FROM {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_RAW_XAPI_TABLE }}; - """ - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py index 048a878f1..24a35eed8 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0023_extend_display_names.py @@ -1,7 +1,7 @@ from alembic import op revision = "0023" -down_revision = "0022" +down_revision = "0020" branch_labels = None depends_on = None on_cluster = ( diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0024_escape_problem_responses.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0024_escape_problem_responses.py deleted file mode 100644 index 320227175..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0024_escape_problem_responses.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -replace double-quoted strings in problem responses -""" - -from alembic import op - -revision = "0024" -down_revision = "0023" -branch_labels = None -depends_on = None -on_cluster = ( - " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " - if "{{CLICKHOUSE_CLUSTER_NAME}}" - else "" -) -engine = ( - "ReplicatedReplacingMergeTree" - if "{{CLICKHOUSE_CLUSTER_NAME}}" - else "ReplacingMergeTree" -) - - -TABLE_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}" -VIEW_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}" - -PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id); -""" - - -OLD_PROBLEM_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -NEW_PROBLEM_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - replaceAll( - replaceAll( - JSON_VALUE(event_str, '$.result.response'), - '\\\'', '\\\\\\\'' - ), '"', '\\\'') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -def drop_objects(): - op.execute( - f""" - DROP TABLE IF EXISTS {TABLE_NAME} {on_cluster} - """ - ) - - op.execute( - f""" - DROP VIEW IF EXISTS {VIEW_NAME} {on_cluster} - """ - ) - - -def upgrade(): - drop_objects() - op.execute(PROBLEM_DDL) - op.execute(f"INSERT INTO {TABLE_NAME} {NEW_PROBLEM_QUERY}") - op.execute( - f""" - CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {NEW_PROBLEM_QUERY} - """ - ) - - -def downgrade(): - drop_objects() - op.execute(PROBLEM_DDL) - op.execute(f"INSERT INTO {TABLE_NAME} {OLD_PROBLEM_QUERY}") - op.execute( - f""" - CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {OLD_PROBLEM_QUERY} - """ - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py deleted file mode 100644 index 7b0663d6e..000000000 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Relies on ERB to do the right thing with responses instead of trying to -escape quotes for lists. -""" - -from alembic import op - -revision = "0025" -down_revision = "0024" -branch_labels = None -depends_on = None -on_cluster = ( - " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " - if "{{CLICKHOUSE_CLUSTER_NAME}}" - else "" -) -engine = ( - "ReplicatedReplacingMergeTree" - if "{{CLICKHOUSE_CLUSTER_NAME}}" - else "ReplacingMergeTree" -) - - -TABLE_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}" -VIEW_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}" - -PROBLEM_DDL = f""" -CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} -{on_cluster} -( - `event_id` UUID NOT NULL, - `emission_time` DateTime NOT NULL, - `actor_id` String NOT NULL, - `object_id` String NOT NULL, - `course_key` String NOT NULL, - `org` String NOT NULL, - `verb_id` LowCardinality(String) NOT NULL, - `responses` String, - `scaled_score` String, - `success` Bool, - `interaction_type` LowCardinality(String), - `attempts` Int16 -) ENGINE = {engine} -PRIMARY KEY (org, course_key, verb_id) -ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id); -""" - - -OLD_PROBLEM_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - replaceAll( - replaceAll( - JSON_VALUE(event_str, '$.result.response'), - '\\\'', '\\\\\\\'' - ), '"', '\\\'') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -NEW_PROBLEM_QUERY = """ -SELECT - event_id, - cast(emission_time as DateTime) as emission_time, - actor_id, - object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id, - JSON_VALUE(event_str, '$.result.response') as responses, - JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.result.success') as Bool), - false - ) as success, - JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, - if( - verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', - cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), - 0 - ) as attempts -FROM - {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} -WHERE - verb_id in ( - 'https://w3id.org/xapi/acrossx/verbs/evaluated', - 'http://adlnet.gov/expapi/verbs/passed', - 'http://adlnet.gov/expapi/verbs/asked' - ); -""" - - -def drop_objects(): - op.execute( - f""" - DROP TABLE IF EXISTS {TABLE_NAME} {on_cluster} - """ - ) - - op.execute( - f""" - DROP VIEW IF EXISTS {VIEW_NAME} {on_cluster} - """ - ) - - -def upgrade(): - drop_objects() - op.execute(PROBLEM_DDL) - op.execute(f"INSERT INTO {TABLE_NAME} {NEW_PROBLEM_QUERY}") - op.execute( - f""" - CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {NEW_PROBLEM_QUERY} - """ - ) - - -def downgrade(): - drop_objects() - op.execute(PROBLEM_DDL) - op.execute(f"INSERT INTO {TABLE_NAME} {OLD_PROBLEM_QUERY}") - op.execute( - f""" - CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {OLD_PROBLEM_QUERY} - """ - ) diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0026_event_sink_external_id.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0026_event_sink_external_id.py index f17f47897..17551ea6a 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0026_event_sink_external_id.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0026_event_sink_external_id.py @@ -5,7 +5,7 @@ revision = "0026" -down_revision = "0025" +down_revision = "0023" branch_labels = None depends_on = None on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else "" diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0027_partition_event_sink_user_profile.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0027_partition_event_sink_user_profile.py index 285d5022b..a62bfa1f8 100644 --- a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0027_partition_event_sink_user_profile.py +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0027_partition_event_sink_user_profile.py @@ -53,10 +53,16 @@ def upgrade(): PARTITION BY user_id MOD 100 PRIMARY KEY (id, time_last_dumped) ORDER BY (id, time_last_dumped) - AS SELECT * FROM {old_user_profile_table} """ ) - # 3. Drop the old table + # 3. Insert data from the old table into the new one + op.execute( + f""" + INSERT INTO {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_USER_PROFILE_TABLE }} + SELECT * FROM {old_user_profile_table} + """ + ) + # 4. Drop the old table op.execute( f""" DROP TABLE {old_user_profile_table} @@ -105,11 +111,17 @@ def downgrade(): ) engine = {engine} PRIMARY KEY (id, time_last_dumped) ORDER BY (id, time_last_dumped) - AS SELECT * FROM {old_user_profile_table} """ ) + # 3. Insert into new table from old one + op.execute( + f""" + INSERT INTO {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_USER_PROFILE_TABLE }} + SELECT * FROM {old_user_profile_table} + """ - # 3. Drop the old table + ) + # 4. Drop the old table op.execute( f""" DROP TABLE {old_user_profile_table} diff --git a/tutoraspects/templates/aspects/build/aspects/requirements.txt b/tutoraspects/templates/aspects/build/aspects/requirements.txt index 9353c8e64..300fe3721 100644 --- a/tutoraspects/templates/aspects/build/aspects/requirements.txt +++ b/tutoraspects/templates/aspects/build/aspects/requirements.txt @@ -2,6 +2,6 @@ alembic==1.11.1 clickhouse-sqlalchemy==0.1.9 # dbt packages -dbt-core==1.4.0 -dbt-clickhouse==1.4.1 +dbt-core==1.7.0 +dbt-clickhouse==1.7.1 git+https://github.com/openedx/xapi-db-load@0.8#egg=xapi-db-load==0.8