Работа с mymsuapi (#62)

## Изменения  ## Детали реализации  ## Check-List  - [x] Вы проверили свой код перед отправкой запроса? - [ ] Вы написали тесты к реализованным функциям? - [x] Вы не забыли применить форматирование `black` и `isort` для _Back-End_ или `Prettier` для _Front-End_? #55
profcomff · Aug 3, 2024 · bd5c0f0 · bd5c0f0
1 parent 8627cad
commit bd5c0f0
Show file tree

Hide file tree

Showing 6 changed files with 322 additions and 0 deletions.
diff --git a/migrations/versions/20240723_1829_0801e032e4e7_my_migration.py b/migrations/versions/20240723_1829_0801e032e4e7_my_migration.py
@@ -5,6 +5,7 @@
 Create Date: 2024-07-23 18:29:37.398287
 
 """
+
 import os
 
 import sqlalchemy as sa

diff --git a/migrations/versions/20240803_1005_ec6f952103aa_schemas_to_store_mymsuapi_data.py b/migrations/versions/20240803_1005_ec6f952103aa_schemas_to_store_mymsuapi_data.py
@@ -0,0 +1,275 @@
+"""schemas to store mymsuapi data
+
+Revision ID: ec6f952103aa
+Revises: 0801e032e4e7
+Create Date: 2024-08-03 10:05:03.253825
+
+"""
+
+import os
+
+import sqlalchemy as sa
+from alembic import op
+
+
+revision = 'ec6f952103aa'
+down_revision = '0801e032e4e7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table_schema("ODS_MYMSUAPI")
+    op.create_table_schema("STG_MYMSUAPI")
+    op.create_table(
+        'ods_timetable_api_flattened',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('group_name', sa.String(), nullable=False),
+        sa.Column('discipline_name', sa.String(), nullable=False),
+        sa.Column('discipline_id', sa.Integer(), nullable=False),
+        sa.Column('classroom_name', sa.String(), nullable=False),
+        sa.Column('classroom_id', sa.Integer(), nullable=False),
+        sa.Column('lesson_type_text', sa.String(), nullable=False),
+        sa.Column('lesson_from_dttm_ts', sa.DateTime(), nullable=False),
+        sa.Column('lesson_to_dttm_ts', sa.DateTime(), nullable=False),
+        sa.Column('teacher_full_name', sa.String(), nullable=False),
+        sa.Column('study_group_id', sa.Integer(), nullable=False),
+        sa.Column('study_group_name', sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        schema='ODS_MYMSUAPI',
+    )
+    op.create_table(
+        'raw_timetable_api',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('group_name', sa.String(), nullable=False),
+        sa.Column('discipline_name', sa.String(), nullable=False),
+        sa.Column('discipline_id', sa.Integer(), nullable=False),
+        sa.Column('classroom_name', sa.String(), nullable=False),
+        sa.Column('classroom_id', sa.Integer(), nullable=False),
+        sa.Column('lesson_type_text', sa.String(), nullable=False),
+        sa.Column('lesson_from_dttm_ts', sa.TIMESTAMP(), nullable=False),
+        sa.Column('lesson_to_dttm_ts', sa.TIMESTAMP(), nullable=False),
+        sa.Column('teacher_users', sa.JSON(), nullable=False),
+        sa.Column('study_groups', sa.JSON(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        schema='STG_MYMSUAPI',
+    )
+    op.create_group(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read"
+    )
+    op.create_group(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write"
+    )
+    op.create_group(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all"
+    )
+    op.create_group(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read"
+    )
+    op.create_group(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write"
+    )
+    op.create_group(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all"
+    )
+    op.grant_on_schema(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        "ODS_MYMSUAPI",
+    )
+    op.grant_on_schema(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        "ODS_MYMSUAPI",
+    )
+    op.grant_on_schema(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        "ODS_MYMSUAPI",
+    )
+    op.grant_on_schema(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        "STG_MYMSUAPI",
+    )
+    op.grant_on_schema(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        "STG_MYMSUAPI",
+    )
+    op.grant_on_schema(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        "STG_MYMSUAPI",
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        ['SELECT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        ['ALL'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        ['SELECT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        ['ALL'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        ['SELECT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        ['ALL'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        ['SELECT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.grant_on_table(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        ['ALL'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.add_column('dim_event_act', sa.Column('source_name', sa.String(), nullable=False), schema='DM_TIMETABLE')
+    op.add_column('dim_group_act', sa.Column('source_name', sa.String(), nullable=False), schema='DM_TIMETABLE')
+    op.add_column('dim_lecturer_act', sa.Column('source_name', sa.String(), nullable=False), schema='DM_TIMETABLE')
+    op.add_column('dim_room_act', sa.Column('source_name', sa.String(), nullable=False), schema='DM_TIMETABLE')
+
+
+def downgrade():
+    op.drop_column('dim_room_act', 'source_name', schema='DM_TIMETABLE')
+    op.drop_column('dim_lecturer_act', 'source_name', schema='DM_TIMETABLE')
+    op.drop_column('dim_group_act', 'source_name', schema='DM_TIMETABLE')
+    op.drop_column('dim_event_act', 'source_name', schema='DM_TIMETABLE')
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        ['ALL'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        ['SELECT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        ['ALL'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        ['SELECT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        ['ALL'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        ['SELECT'],
+        '"STG_MYMSUAPI".raw_timetable_api',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        ['ALL'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_table(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        ['SELECT'],
+        '"ODS_MYMSUAPI".ods_timetable_api_flattened',
+    )
+    op.revoke_on_schema(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all",
+        "STG_MYMSUAPI",
+    )
+    op.revoke_on_schema(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write",
+        "STG_MYMSUAPI",
+    )
+    op.revoke_on_schema(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read",
+        "STG_MYMSUAPI",
+    )
+    op.revoke_on_schema(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all",
+        "ODS_MYMSUAPI",
+    )
+    op.revoke_on_schema(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write",
+        "ODS_MYMSUAPI",
+    )
+    op.revoke_on_schema(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read",
+        "ODS_MYMSUAPI",
+    )
+    op.drop_table('raw_timetable_api', schema='STG_MYMSUAPI')
+    op.drop_table('ods_timetable_api_flattened', schema='ODS_MYMSUAPI')
+    op.delete_group(
+        "test_dwh_stg_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_all"
+    )
+    op.delete_group(
+        "test_dwh_stg_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_write"
+    )
+    op.delete_group(
+        "test_dwh_stg_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_mymsuapi_read"
+    )
+    op.delete_group(
+        "test_dwh_ods_mymsuapi_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_all"
+    )
+    op.delete_group(
+        "test_dwh_ods_mymsuapi_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_write"
+    )
+    op.delete_group(
+        "test_dwh_ods_mymsuapi_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_mymsuapi_read"
+    )
+    op.drop_table_schema("STG_MYMSUAPI")
+    op.drop_table_schema("ODS_MYMSUAPI")
diff --git a/profcomff_definitions/DM/timetable.py b/profcomff_definitions/DM/timetable.py
@@ -10,6 +10,7 @@ class DimRoomAct(Base):
     room_direction_text_type: Mapped[str | None]
     room_is_deleted: Mapped[bool]
     room_department: Mapped[str | None]
+    source_name: Mapped[str]
 
 
 class DimLecturerAct(Base):
@@ -20,15 +21,18 @@ class DimLecturerAct(Base):
     lecturer_last_name: Mapped[str | None]
     lecturer_avatar_id: Mapped[int | None]
     lecturer_description: Mapped[str | None]
+    source_name: Mapped[str]
 
 
 class DimGroupAct(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     group_api_id: Mapped[int]
     group_name_text: Mapped[str | None]
     group_number: Mapped[int | None]
+    source_name: Mapped[str]
 
 
 class DimEventAct(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     event_name_text: Mapped[str | None]
+    source_name: Mapped[str]
diff --git a/profcomff_definitions/ODS/info.py b/profcomff_definitions/ODS/info.py
@@ -3,6 +3,7 @@
 
 from profcomff_definitions.base import Base
 
+
 class Info(Base):
     user_id: Mapped[int] = mapped_column(Integer, index=True, primary_key=True)
     email: Mapped[str | None] = mapped_column(String, nullable=True)

diff --git a/profcomff_definitions/ODS/mymsuapi.py b/profcomff_definitions/ODS/mymsuapi.py
@@ -0,0 +1,21 @@
+from datetime import datetime
+
+from sqlalchemy import TIMESTAMP
+from sqlalchemy.orm import Mapped, mapped_column
+
+from profcomff_definitions.base import Base
+
+
+class OdsTimetableApiFlattened(Base):
+    id: Mapped[int] = mapped_column(primary_key=True)
+    group_name: Mapped[str]
+    discipline_name: Mapped[str]
+    discipline_id: Mapped[int]
+    classroom_name: Mapped[str]
+    classroom_id: Mapped[int]
+    lesson_type_text: Mapped[str]
+    lesson_from_dttm_ts: Mapped[datetime]
+    lesson_to_dttm_ts: Mapped[datetime]
+    teacher_full_name: Mapped[str]
+    study_group_id: Mapped[int]
+    study_group_name: Mapped[str]
diff --git a/profcomff_definitions/STG/mymsuapi.py b/profcomff_definitions/STG/mymsuapi.py
@@ -0,0 +1,20 @@
+from datetime import datetime
+
+from sqlalchemy import JSON, TIMESTAMP
+from sqlalchemy.orm import Mapped, mapped_column
+
+from profcomff_definitions.base import Base
+
+
+class RawTimetableApi(Base):
+    id: Mapped[int] = mapped_column(primary_key=True)
+    group_name: Mapped[str]
+    discipline_name: Mapped[str]
+    discipline_id: Mapped[int]
+    classroom_name: Mapped[str]
+    classroom_id: Mapped[int]
+    lesson_type_text: Mapped[str]
+    lesson_from_dttm_ts: Mapped[datetime] = mapped_column(TIMESTAMP)
+    lesson_to_dttm_ts: Mapped[datetime] = mapped_column(TIMESTAMP)
+    teacher_users: Mapped[dict] = mapped_column(JSON)
+    study_groups: Mapped[dict] = mapped_column(JSON)
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ @@
     Create Date: 2024-07-23 18:29:37.398287
     """
     import os
     import sqlalchemy as sa
@@ Expand Down @@