From 13ea7e3b07208e20246b61380e820b4af6c94505 Mon Sep 17 00:00:00 2001 From: SofyaFin <151746034+SofyaFin@users.noreply.github.com> Date: Tue, 25 Jun 2024 14:03:40 +0300 Subject: [PATCH] #44 (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Изменения 44 на нужной ветке ## Детали реализации ## Check-List - [ ] Вы проверили свой код перед отправкой запроса? - [ ] Вы написали тесты к реализованным функциям? - [ ] Вы не забыли применить форматирование `black` и `isort` для _Back-End_ или `Prettier` для _Front-End_? --- ..._1330_8b6f4a99f692_add_dimroomact_to_dm.py | 6 +- .../versions/20240512_0913_7353088d02ce_.py | 5 +- ...2_7a18bd9ff633_github_stg_new_repo_cols.py | 8 +-- ...0240625_1206_5714921e11a0_incident_logs.py | 72 +++++++++++++++++++ profcomff_definitions/DM/infra_logs.py | 12 ++++ tests/conftest.py | 9 +-- 6 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 migrations/versions/20240625_1206_5714921e11a0_incident_logs.py diff --git a/migrations/versions/20240510_1330_8b6f4a99f692_add_dimroomact_to_dm.py b/migrations/versions/20240510_1330_8b6f4a99f692_add_dimroomact_to_dm.py index 1211102e..d037b0b8 100644 --- a/migrations/versions/20240510_1330_8b6f4a99f692_add_dimroomact_to_dm.py +++ b/migrations/versions/20240510_1330_8b6f4a99f692_add_dimroomact_to_dm.py @@ -6,10 +6,12 @@ """ -from alembic import op -import sqlalchemy as sa import os +import sqlalchemy as sa +from alembic import op + + # revision identifiers, used by Alembic. revision = '8b6f4a99f692' down_revision = 'd459997cd681' diff --git a/migrations/versions/20240512_0913_7353088d02ce_.py b/migrations/versions/20240512_0913_7353088d02ce_.py index 05706176..43c0c31b 100644 --- a/migrations/versions/20240512_0913_7353088d02ce_.py +++ b/migrations/versions/20240512_0913_7353088d02ce_.py @@ -6,10 +6,11 @@ """ -from alembic import op -import sqlalchemy as sa import os +import sqlalchemy as sa +from alembic import op + # revision identifiers, used by Alembic. revision = '7353088d02ce' diff --git a/migrations/versions/20240513_1902_7a18bd9ff633_github_stg_new_repo_cols.py b/migrations/versions/20240513_1902_7a18bd9ff633_github_stg_new_repo_cols.py index d228e6d2..4280ac3d 100644 --- a/migrations/versions/20240513_1902_7a18bd9ff633_github_stg_new_repo_cols.py +++ b/migrations/versions/20240513_1902_7a18bd9ff633_github_stg_new_repo_cols.py @@ -38,15 +38,11 @@ def upgrade(): sa.Column('security_and_analysis_secret_scanning_validity_checks_status', sa.String(), nullable=True), schema='STG_GITHUB', ) - op.alter_column( - 'profcomff_issue', 'id', existing_type=sa.INTEGER(), type_=sa.BIGINT(), schema='STG_GITHUB' - ) + op.alter_column('profcomff_issue', 'id', existing_type=sa.INTEGER(), type_=sa.BIGINT(), schema='STG_GITHUB') def downgrade(): - op.alter_column( - 'profcomff_issue', 'id', existing_type=sa.BIGINT(), type_=sa.INTEGER(), schema='STG_GITHUB' - ) + op.alter_column('profcomff_issue', 'id', existing_type=sa.BIGINT(), type_=sa.INTEGER(), schema='STG_GITHUB') op.drop_column( 'profcomff_repo', 'security_and_analysis_secret_scanning_validity_checks_status', schema='STG_GITHUB' ) diff --git a/migrations/versions/20240625_1206_5714921e11a0_incident_logs.py b/migrations/versions/20240625_1206_5714921e11a0_incident_logs.py new file mode 100644 index 00000000..19be2a35 --- /dev/null +++ b/migrations/versions/20240625_1206_5714921e11a0_incident_logs.py @@ -0,0 +1,72 @@ +"""incident_logs + +Revision ID: 5714921e11a0 +Revises: 7a18bd9ff633 +Create Date: 2024-06-25 12:06:54.755277 + +""" + +import os + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '5714921e11a0' +down_revision = '7a18bd9ff633' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'incident_hint', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('msk_record_loaded_dttm', sa.DateTime(), nullable=False, comment='Поле нарезки лога'), + sa.Column( + 'container_name', sa.String(), nullable=False, comment='Имя контейнера, в котором произошла ошибочка' + ), + sa.Column('message', sa.String(), nullable=False, comment='Сообщение об ошибке'), + sa.Column('create_ts', sa.DateTime(), nullable=False, comment='Время, когда произошла ошибка'), + sa.PrimaryKeyConstraint('id'), + schema='DM_INFRA_LOGS', + comment='Информация об ошибках по контейнерам', + ) + op.grant_on_table( + "test_dwh_dm_infra_logs_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_read", + ['SELECT'], + '"DM_INFRA_LOGS".incident_hint', + ) + op.grant_on_table( + "test_dwh_dm_infra_logs_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_write", + ['SELECT', 'INSERT', 'DELETE', 'UPDATE', 'TRUNCATE'], + '"DM_INFRA_LOGS".incident_hint', + ) + op.grant_on_table( + "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all", + ['ALL'], + '"DM_INFRA_LOGS".incident_hint', + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.revoke_on_table( + "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all", + ['ALL'], + '"DM_INFRA_LOGS".incident_hint', + ) + op.revoke_on_table( + "test_dwh_dm_infra_logs_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_write", + ['SELECT', 'INSERT', 'DELETE', 'UPDATE', 'TRUNCATE'], + '"DM_INFRA_LOGS".incident_hint', + ) + op.revoke_on_table( + "test_dwh_dm_infra_logs_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_read", + ['SELECT'], + '"DM_INFRA_LOGS".incident_hint', + ) + op.drop_table('incident_hint', schema='DM_INFRA_LOGS') diff --git a/profcomff_definitions/DM/infra_logs.py b/profcomff_definitions/DM/infra_logs.py index ceb706b8..8be88257 100644 --- a/profcomff_definitions/DM/infra_logs.py +++ b/profcomff_definitions/DM/infra_logs.py @@ -1,3 +1,5 @@ +from datetime import datetime + from sqlalchemy.orm import Mapped, mapped_column from profcomff_definitions.base import Base @@ -15,3 +17,13 @@ class ContainerLogCube(Base): error_cnt: Mapped[int] = mapped_column(comment="Количество записей с типом ERROR") critical_cnt: Mapped[int] = mapped_column(comment="Количество записей с типом CRITICAL") other_cnt: Mapped[int] = mapped_column(comment="Количество записей с другими типами") + + +class IncidentHint(Base): + """Информация об ошибках по контейнерам""" + + id: Mapped[int] = mapped_column(primary_key=True) + msk_record_loaded_dttm: Mapped[datetime] = mapped_column(comment="Поле нарезки лога") + container_name: Mapped[str] = mapped_column(comment="Имя контейнера, в котором произошла ошибочка") + message: Mapped[str] = mapped_column(comment="Сообщение об ошибке") + create_ts: Mapped[datetime] = mapped_column(comment="Время, когда произошла ошибка") diff --git a/tests/conftest.py b/tests/conftest.py index e62071c8..9ba8a844 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,11 +3,10 @@ from typing import Generator import pytest -from alembic.config import Config -from sqlalchemy import create_engine -from alembic.command import downgrade, upgrade, revision +from alembic.command import downgrade, revision, upgrade from alembic.config import Config from alembic.script import Script, ScriptDirectory +from sqlalchemy import create_engine from sqlalchemy.engine import Engine @@ -18,7 +17,9 @@ def alembic_config(): alembic_cfg = Config() alembic_cfg.set_main_option('script_location', str(REPO_ROOT / "migrations")) - alembic_cfg.set_main_option('sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres") # db for migration tests + alembic_cfg.set_main_option( + 'sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres" + ) # db for migration tests return alembic_cfg