From 060e040d66e20ec582acbf668b81b009525eae78 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Sun, 14 Apr 2024 23:23:08 +0300 Subject: [PATCH 01/12] on_event -> lifespan --- social/routes/base.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/social/routes/base.py b/social/routes/base.py index 4729708..c07e967 100644 --- a/social/routes/base.py +++ b/social/routes/base.py @@ -13,16 +13,29 @@ settings = get_settings() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + telegram = get_telegram() + await telegram.initialize() + await telegram.start() + yield + # Clean up the ML models and release the resources + await telegram.stop() + await telegram.shutdown() + + app = FastAPI( title='Сервис мониторинга активности', description=('Серверная часть сервиса для выдачи печенек за активности'), version=__version__, + lifespan=lifespan, # Настраиваем интернет документацию root_path=settings.ROOT_PATH if __version__ != 'dev' else '/', docs_url=None if __version__ != 'dev' else '/docs', redoc_url=None, ) -telegram = get_telegram() app.add_middleware( @@ -40,18 +53,7 @@ ) -@app.on_event("startup") -async def startup(): - await telegram.initialize() - await telegram.start() - - -@app.on_event("shutdown") -async def shutdown(): - await telegram.stop() - await telegram.shutdown() - - +app.include_router(group_router) app.include_router(github_router) app.include_router(telegram_router) app.include_router(vk_router) From bf65b25bd3e04bd4b36502d95c00b0d38300a7ac Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:03:39 +0300 Subject: [PATCH 02/12] Add group types --- .../1cacaf803a1d_user_defined_groups.py | 88 +++++++++++++++++++ social/models/__init__.py | 4 +- social/models/group.py | 61 +++++++++++++ social/models/vk.py | 15 ---- social/routes/vk.py | 8 +- 5 files changed, 155 insertions(+), 21 deletions(-) create mode 100644 migrations/versions/1cacaf803a1d_user_defined_groups.py create mode 100644 social/models/group.py delete mode 100644 social/models/vk.py diff --git a/migrations/versions/1cacaf803a1d_user_defined_groups.py b/migrations/versions/1cacaf803a1d_user_defined_groups.py new file mode 100644 index 0000000..5d41a9d --- /dev/null +++ b/migrations/versions/1cacaf803a1d_user_defined_groups.py @@ -0,0 +1,88 @@ +"""User defined groups + +Revision ID: 1cacaf803a1d +Revises: 9d98c1e9c864 +Create Date: 2024-04-14 23:38:18.956845 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '1cacaf803a1d' +down_revision = '9d98c1e9c864' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'group', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('type', sa.String(), nullable=False), + sa.Column('owner_id', sa.Integer(), nullable=True), + sa.Column('is_deleted', sa.Boolean(), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('activation_token', sa.String(), nullable=True), + sa.Column('create_ts', sa.DateTime(), nullable=False), + sa.Column('update_ts', sa.DateTime(), nullable=False), + ) + op.execute(''' + INSERT INTO "group" + (id, type, is_deleted, is_active, create_ts, update_ts) + SELECT id, 'vk_group', False, False, create_ts, update_ts + FROM vk_groups; + ''') + op.create_primary_key('group_pk', 'group', ['id']) + op.create_table( + 'telegram_channel', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('channel_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ['id'], + ['group.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_table( + 'telegram_chat', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ['id'], + ['group.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_table( + 'vk_chat', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ['id'], + ['group.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_foreign_key('group_vkgroup_fk', 'vk_groups', 'group', ['id'], ['id']) + op.drop_column('vk_groups', 'update_ts') + op.drop_column('vk_groups', 'create_ts') + op.execute('DROP SEQUENCE IF EXISTS vk_groups_id_seq;') + op.rename_table('vk_groups', 'vk_group') + + +def downgrade(): + op.rename_table('vk_group', 'vk_groups') + op.add_column('vk_groups', sa.Column('create_ts', sa.DateTime())) + op.add_column('vk_groups', sa.Column('update_ts', sa.DateTime())) + op.execute('UPDATE vk_groups SET create_ts = (SELECT create_ts FROM "group" WHERE "group".id = vk_groups.id);') + op.execute('UPDATE vk_groups SET update_ts = (SELECT update_ts FROM "group" WHERE "group".id = vk_groups.id);') + op.alter_column('vk_groups', 'create_ts', nullable=False) + op.alter_column('vk_groups', 'update_ts', nullable=False) + op.drop_constraint('group_vkgroup_fk', 'vk_groups', type_='foreignkey') + op.drop_table('vk_chat') + op.drop_table('telegram_chat') + op.drop_table('telegram_channel') + op.drop_table('group') diff --git a/social/models/__init__.py b/social/models/__init__.py index 0ff4812..471545b 100644 --- a/social/models/__init__.py +++ b/social/models/__init__.py @@ -1,5 +1,5 @@ -from .vk import VkGroups +from .group import TelegramChannel, TelegramChat, VkGroup, VkChat from .webhook_storage import WebhookStorage -__all__ = ['WebhookStorage', 'VkGroups'] +__all__ = ['WebhookStorage', 'TelegramChannel', 'TelegramChat', 'VkGroup', 'VkChat'] diff --git a/social/models/group.py b/social/models/group.py new file mode 100644 index 0000000..5ffd33e --- /dev/null +++ b/social/models/group.py @@ -0,0 +1,61 @@ +from datetime import datetime, UTC + +import sqlalchemy as sa +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class Group(Base): + id: Mapped[int] = mapped_column(primary_key=True) + type: Mapped[str] + owner_id: Mapped[int | None] + + is_deleted: Mapped[bool] = mapped_column(default=False) + is_active: Mapped[bool] = mapped_column(default=False) + activation_token: Mapped[str | None] + + create_ts: Mapped[datetime] = mapped_column(default=lambda: datetime.now(UTC)) + update_ts: Mapped[datetime] = mapped_column(default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) + + __mapper_args__ = { + "polymorphic_on": "type", + } + + +class VkGroup(Group): + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + group_id: Mapped[int] + confirmation_token: Mapped[str] + secret_key: Mapped[str] + + __mapper_args__ = { + "polymorphic_identity": "vk_group", + } + + +class VkChat(Group): + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + chat_id: Mapped[int] + + __mapper_args__ = { + "polymorphic_identity": "vk_chat", + } + + +class TelegramChannel(Group): + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + channel_id: Mapped[int] + + __mapper_args__ = { + "polymorphic_identity": "tg_channel", + } + + +class TelegramChat(Group): + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + chat_id: Mapped[int] + + __mapper_args__ = { + "polymorphic_identity": "tg_chat", + } diff --git a/social/models/vk.py b/social/models/vk.py deleted file mode 100644 index da72616..0000000 --- a/social/models/vk.py +++ /dev/null @@ -1,15 +0,0 @@ -from datetime import datetime - -import sqlalchemy as sa -from sqlalchemy.orm import Mapped, mapped_column - -from .base import Base - - -class VkGroups(Base): - id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) - group_id: Mapped[int] = mapped_column(sa.Integer) - confirmation_token: Mapped[str] = mapped_column(sa.String) - secret_key: Mapped[str] = mapped_column(sa.String) - create_ts: Mapped[datetime] = mapped_column(sa.DateTime, default=datetime.utcnow) - update_ts: Mapped[datetime] = mapped_column(sa.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) diff --git a/social/routes/vk.py b/social/routes/vk.py index 217ddd3..34e7046 100644 --- a/social/routes/vk.py +++ b/social/routes/vk.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, ConfigDict from social.handlers_telegram import get_application -from social.models.vk import VkGroups +from social.models.group import VkGroup from social.models.webhook_storage import WebhookStorage, WebhookSystems from social.settings import get_settings from social.utils.string import random_string @@ -37,7 +37,7 @@ async def vk_webhook(request: Request) -> str: request_data = await request.json() logger.debug(request_data) group_id = request_data["group_id"] # Fail if no group - group = db.session.query(VkGroups).where(VkGroups.group_id == group_id).one() # Fail if no settings + group = db.session.query(VkGroup).where(VkGroup.group_id == group_id).one() # Fail if no settings # Проверка на создание нового вебхука со страничка ВК if request_data.get("type", "") == "confirmation": @@ -61,9 +61,9 @@ async def vk_webhook(request: Request) -> str: def create_or_replace_group( group_id: int, group_info: VkGroupCreate, _=Depends(UnionAuth(["social.vk_group.create"])) ) -> VkGroupCreateResponse: - group = db.session.query(VkGroups).where(VkGroups.group_id == group_id).one_or_none() + group = db.session.query(VkGroup).where(VkGroup.group_id == group_id).one_or_none() if group is None: - group = VkGroups() + group = VkGroup() db.session.add(group) group.group_id = group_id group.secret_key = random_string(32) From 223955264bcdc50c46e3bf41c70e7aadf2d942a1 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:22:18 +0300 Subject: [PATCH 03/12] WebhookStorage event_ts --- .../62addefd9655_webhookstorage_event_ts.py | 25 +++++++++++++++++++ social/models/webhook_storage.py | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 migrations/versions/62addefd9655_webhookstorage_event_ts.py diff --git a/migrations/versions/62addefd9655_webhookstorage_event_ts.py b/migrations/versions/62addefd9655_webhookstorage_event_ts.py new file mode 100644 index 0000000..24cdc99 --- /dev/null +++ b/migrations/versions/62addefd9655_webhookstorage_event_ts.py @@ -0,0 +1,25 @@ +"""WebhookStorage event_ts + +Revision ID: 62addefd9655 +Revises: 1cacaf803a1d +Create Date: 2024-04-15 00:21:54.075449 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '62addefd9655' +down_revision = '1cacaf803a1d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('webhook_storage', sa.Column('event_ts', sa.DateTime(), nullable=True)) + + +def downgrade(): + op.drop_column('webhook_storage', 'event_ts') diff --git a/social/models/webhook_storage.py b/social/models/webhook_storage.py index 590b411..586f9fc 100644 --- a/social/models/webhook_storage.py +++ b/social/models/webhook_storage.py @@ -1,3 +1,4 @@ +from datetime import UTC, datetime from enum import Enum import sqlalchemy as sa @@ -17,3 +18,4 @@ class WebhookStorage(Base): id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) system: Mapped[WebhookSystems] = mapped_column(sa.Enum(WebhookSystems, native_enum=False)) message: Mapped[sa.JSON] = mapped_column(sa.JSON(True)) + event_ts: Mapped[datetime] = mapped_column(default=lambda: datetime.now(UTC), nullable=True) From 790bddd10321cacbe09e21aae88fb6975978c203 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:23:11 +0300 Subject: [PATCH 04/12] Import --- social/routes/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/social/routes/base.py b/social/routes/base.py index c07e967..8c3b9c7 100644 --- a/social/routes/base.py +++ b/social/routes/base.py @@ -1,3 +1,5 @@ +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi_sqlalchemy import DBSessionMiddleware From 9bd9db59d800b8710c58d03847937f5e771325ea Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:50:14 +0300 Subject: [PATCH 05/12] last_active_ts --- .../versions/e26e42fb60e3_last_access.py | 29 +++++++++++++++++++ social/models/group.py | 3 +- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/e26e42fb60e3_last_access.py diff --git a/migrations/versions/e26e42fb60e3_last_access.py b/migrations/versions/e26e42fb60e3_last_access.py new file mode 100644 index 0000000..c053ea0 --- /dev/null +++ b/migrations/versions/e26e42fb60e3_last_access.py @@ -0,0 +1,29 @@ +"""Last access + +Revision ID: e26e42fb60e3 +Revises: 62addefd9655 +Create Date: 2024-04-15 00:43:27.419968 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e26e42fb60e3' +down_revision = '62addefd9655' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('group', sa.Column('last_active_ts', sa.DateTime(), nullable=True)) + op.drop_column('group', 'activation_token') + op.drop_column('group', 'is_active') + + +def downgrade(): + op.add_column('group', sa.Column('is_active', sa.BOOLEAN(), autoincrement=False, nullable=False)) + op.add_column('group', sa.Column('activation_token', sa.VARCHAR(), autoincrement=False, nullable=True)) + op.drop_column('group', 'last_active_ts') diff --git a/social/models/group.py b/social/models/group.py index 5ffd33e..407fdc6 100644 --- a/social/models/group.py +++ b/social/models/group.py @@ -12,8 +12,7 @@ class Group(Base): owner_id: Mapped[int | None] is_deleted: Mapped[bool] = mapped_column(default=False) - is_active: Mapped[bool] = mapped_column(default=False) - activation_token: Mapped[str | None] + last_active_ts: Mapped[datetime | None] create_ts: Mapped[datetime] = mapped_column(default=lambda: datetime.now(UTC)) update_ts: Mapped[datetime] = mapped_column(default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)) From b55cfe5ba03aa5d4c2e908c10228db0634b7ffc2 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:51:22 +0300 Subject: [PATCH 06/12] Do not fail on no token --- social/handlers_telegram/base.py | 2 ++ social/routes/base.py | 14 +++++++------- social/routes/discord.py | 2 +- social/utils/github_api.py | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/social/handlers_telegram/base.py b/social/handlers_telegram/base.py index 0a3aa20..c33d0ec 100644 --- a/social/handlers_telegram/base.py +++ b/social/handlers_telegram/base.py @@ -17,6 +17,8 @@ @lru_cache() def get_application(): + if not settings.TELEGRAM_BOT_TOKEN: + return None context_types = ContextTypes(context=CustomContext) app = Application.builder().token(settings.TELEGRAM_BOT_TOKEN).updater(None).context_types(context_types).build() logger.info("Telegram API initialized successfully") diff --git a/social/routes/base.py b/social/routes/base.py index 8c3b9c7..3ed7385 100644 --- a/social/routes/base.py +++ b/social/routes/base.py @@ -20,12 +20,13 @@ @asynccontextmanager async def lifespan(app: FastAPI): telegram = get_telegram() - await telegram.initialize() - await telegram.start() + if telegram: + await telegram.initialize() + await telegram.start() yield - # Clean up the ML models and release the resources - await telegram.stop() - await telegram.shutdown() + if telegram: + await telegram.stop() + await telegram.shutdown() app = FastAPI( @@ -34,7 +35,7 @@ async def lifespan(app: FastAPI): version=__version__, lifespan=lifespan, # Настраиваем интернет документацию - root_path=settings.ROOT_PATH if __version__ != 'dev' else '/', + root_path=settings.ROOT_PATH if __version__ != 'dev' else '', docs_url=None if __version__ != 'dev' else '/docs', redoc_url=None, ) @@ -55,7 +56,6 @@ async def lifespan(app: FastAPI): ) -app.include_router(group_router) app.include_router(github_router) app.include_router(telegram_router) app.include_router(vk_router) diff --git a/social/routes/discord.py b/social/routes/discord.py index 9048e35..3928418 100644 --- a/social/routes/discord.py +++ b/social/routes/discord.py @@ -14,7 +14,6 @@ router = APIRouter(prefix="/discord", tags=["webhooks"]) settings = get_settings() logger = logging.getLogger(__name__) -verify_key = VerifyKey(bytes.fromhex(settings.DISCORD_PUBLIC_KEY)) @router.post('') @@ -28,6 +27,7 @@ async def discord_webhook(request: Request, background_tasks: BackgroundTasks): body = (await request.body()).decode("utf-8") try: + verify_key = VerifyKey(bytes.fromhex(settings.DISCORD_PUBLIC_KEY)) verify_key.verify(f'{timestamp}{body}'.encode(), bytes.fromhex(signature)) except BadSignatureError: raise HTTPException(401, 'invalid request signature') diff --git a/social/utils/github_api.py b/social/utils/github_api.py index 8829800..181ff2c 100644 --- a/social/utils/github_api.py +++ b/social/utils/github_api.py @@ -114,5 +114,7 @@ def request_gql(self, file_or_query, operation_name, **params): @lru_cache() def get_github(org): + if not settings.GITHUB_APP_ID: + return None github = GitHub(settings.GITHUB_APP_ID, settings.GITHUB_PRIVATE_KEY, org) return github From c2495e854751c2e14bd25aced33f07922c621145 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 00:51:35 +0300 Subject: [PATCH 07/12] Tg autoregister --- social/routes/telegram.py | 40 ++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/social/routes/telegram.py b/social/routes/telegram.py index 906e876..5146ea0 100644 --- a/social/routes/telegram.py +++ b/social/routes/telegram.py @@ -1,4 +1,6 @@ +from datetime import UTC, datetime import logging +from asyncio import create_task from fastapi import APIRouter, Request from fastapi_sqlalchemy import db @@ -6,6 +8,7 @@ from social.handlers_telegram import get_application from social.models.webhook_storage import WebhookStorage, WebhookSystems +from social.models import TelegramChannel, TelegramChat from social.settings import get_settings @@ -21,13 +24,36 @@ async def telegram_webhook(request: Request): request_data = await request.json() logger.debug(request_data) - db.session.add( - WebhookStorage( - system=WebhookSystems.TELEGRAM, - message=request_data, + with db.session as s: + s.add( + WebhookStorage( + system=WebhookSystems.TELEGRAM, + message=request_data, + ) ) - ) - db.session.commit() - await application.update_queue.put(Update.de_json(data=request_data, bot=application.bot)) + update = Update.de_json(data=request_data, bot=application.bot) + add_msg = create_task(application.update_queue.put(update)) + try: + chat = update.effective_chat + obj = None + if chat.type in ['group', 'supergroup']: + obj = db.session.query(TelegramChat).where(TelegramChat.chat_id == chat.id).one_or_none() + if obj is None: + obj = TelegramChat(chat_id=chat.id) + db.session.add(obj) + elif chat.type == 'channel': + obj = db.session.query(TelegramChannel).where(TelegramChannel.channel_id == chat.id).one_or_none() + if obj is None: + obj = TelegramChannel(channel_id=chat.id) + db.session.add(obj) + + obj.last_active_ts = datetime.now(UTC) + db.session.commit() + logger.debug(obj) + except Exception as exc: + logger.exception(exc) + finally: + await add_msg + return From 6275871530c4645c1b931310dc0702a5540b3f75 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 01:26:28 +0300 Subject: [PATCH 08/12] Fix migrations --- .../1cacaf803a1d_user_defined_groups.py | 26 ++++++++++++----- .../versions/e26e42fb60e3_last_access.py | 29 ------------------- 2 files changed, 18 insertions(+), 37 deletions(-) delete mode 100644 migrations/versions/e26e42fb60e3_last_access.py diff --git a/migrations/versions/1cacaf803a1d_user_defined_groups.py b/migrations/versions/1cacaf803a1d_user_defined_groups.py index 5d41a9d..0c058a9 100644 --- a/migrations/versions/1cacaf803a1d_user_defined_groups.py +++ b/migrations/versions/1cacaf803a1d_user_defined_groups.py @@ -8,7 +8,7 @@ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql +from sqlalchemy.schema import Sequence, CreateSequence # revision identifiers, used by Alembic. revision = '1cacaf803a1d' @@ -24,22 +24,26 @@ def upgrade(): sa.Column('type', sa.String(), nullable=False), sa.Column('owner_id', sa.Integer(), nullable=True), sa.Column('is_deleted', sa.Boolean(), nullable=False), - sa.Column('is_active', sa.Boolean(), nullable=False), - sa.Column('activation_token', sa.String(), nullable=True), + sa.Column('last_active_ts', sa.DateTime(), nullable=False), sa.Column('create_ts', sa.DateTime(), nullable=False), sa.Column('update_ts', sa.DateTime(), nullable=False), ) op.execute(''' INSERT INTO "group" - (id, type, is_deleted, is_active, create_ts, update_ts) - SELECT id, 'vk_group', False, False, create_ts, update_ts + (id, type, is_deleted, last_active_ts, create_ts, update_ts) + SELECT id, 'vk_group', False, now(), create_ts, update_ts FROM vk_groups; ''') + + max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "group";')).scalar() op.create_primary_key('group_pk', 'group', ['id']) + op.execute(CreateSequence(Sequence('group_id_seq', max_id+1))) + op.alter_column('group', 'id', server_default=sa.text('nextval(\'group_id_seq\')')) + op.create_table( 'telegram_channel', sa.Column('id', sa.Integer(), nullable=False), - sa.Column('channel_id', sa.Integer(), nullable=False), + sa.Column('channel_id', sa.BigInteger(), nullable=False), sa.ForeignKeyConstraint( ['id'], ['group.id'], @@ -49,7 +53,7 @@ def upgrade(): op.create_table( 'telegram_chat', sa.Column('id', sa.Integer(), nullable=False), - sa.Column('chat_id', sa.Integer(), nullable=False), + sa.Column('chat_id', sa.BigInteger(), nullable=False), sa.ForeignKeyConstraint( ['id'], ['group.id'], @@ -69,12 +73,17 @@ def upgrade(): op.create_foreign_key('group_vkgroup_fk', 'vk_groups', 'group', ['id'], ['id']) op.drop_column('vk_groups', 'update_ts') op.drop_column('vk_groups', 'create_ts') - op.execute('DROP SEQUENCE IF EXISTS vk_groups_id_seq;') + op.execute('DROP SEQUENCE IF EXISTS vk_groups_id_seq CASCADE;') op.rename_table('vk_groups', 'vk_group') def downgrade(): op.rename_table('vk_group', 'vk_groups') + + max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "vk_groups";')).scalar() + op.execute(CreateSequence(Sequence('vk_groups_id_seq', max_id+1))) + op.alter_column('vk_groups', 'id', server_default=sa.text('nextval(\'vk_groups_id_seq\')')) + op.add_column('vk_groups', sa.Column('create_ts', sa.DateTime())) op.add_column('vk_groups', sa.Column('update_ts', sa.DateTime())) op.execute('UPDATE vk_groups SET create_ts = (SELECT create_ts FROM "group" WHERE "group".id = vk_groups.id);') @@ -86,3 +95,4 @@ def downgrade(): op.drop_table('telegram_chat') op.drop_table('telegram_channel') op.drop_table('group') + op.execute('DROP SEQUENCE IF EXISTS group_id_seq CASCADE;') diff --git a/migrations/versions/e26e42fb60e3_last_access.py b/migrations/versions/e26e42fb60e3_last_access.py deleted file mode 100644 index c053ea0..0000000 --- a/migrations/versions/e26e42fb60e3_last_access.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Last access - -Revision ID: e26e42fb60e3 -Revises: 62addefd9655 -Create Date: 2024-04-15 00:43:27.419968 - -""" - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'e26e42fb60e3' -down_revision = '62addefd9655' -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column('group', sa.Column('last_active_ts', sa.DateTime(), nullable=True)) - op.drop_column('group', 'activation_token') - op.drop_column('group', 'is_active') - - -def downgrade(): - op.add_column('group', sa.Column('is_active', sa.BOOLEAN(), autoincrement=False, nullable=False)) - op.add_column('group', sa.Column('activation_token', sa.VARCHAR(), autoincrement=False, nullable=True)) - op.drop_column('group', 'last_active_ts') From 555f428bd8f8b70a945e46ddcb9157362781a406 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 01:37:19 +0300 Subject: [PATCH 09/12] Add checks action --- .github/workflows/checks.yml | 78 ++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/checks.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..a6c76e6 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,78 @@ +name: Python tests + +on: + pull_request: + +jobs: + test: + name: Unit tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_HOST_AUTH_METHOD: trust + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + -p 5432:5432 + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m ensurepip + python -m pip install --upgrade pip + pip install --upgrade -r requirements.txt -r requirements.dev.txt + - name: Migrate DB + run: | + DB_DSN=postgresql://postgres@localhost:5432/postgres alembic upgrade head + - name: Build coverage file + id: pytest + run: | + DB_DSN=postgresql://postgres@localhost:5432/postgres pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=services_backend tests/ | tee pytest-coverage.txt + exit ${PIPESTATUS[0]} + - name: Print report + if: always() + run: | + cat pytest-coverage.txt + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@main + with: + pytest-coverage-path: ./pytest-coverage.txt + title: Coverage Report + badge-title: Code Coverage + hide-badge: false + hide-report: false + create-new-comment: false + hide-comment: false + report-only-changed-files: false + remove-link-from-badge: false + junitxml-path: ./pytest.xml + junitxml-title: Summary + - name: Fail on pytest errors + if: steps.pytest.outcome == 'failure' + run: exit 1 + + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v2 + with: + python-version: 3.11 + - uses: isort/isort-action@master + with: + requirementsFiles: "requirements.txt requirements.dev.txt" + - uses: psf/black@stable + - name: Comment if linting failed + if: failure() + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + :poop: Code linting failed, use `black` and `isort` to fix it. From 1db28cdde92abe17b1319ec1b681dba280929fca Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 01:40:06 +0300 Subject: [PATCH 10/12] Linting --- Makefile | 6 ++++ .../1cacaf803a1d_user_defined_groups.py | 15 +++++---- .../versions/57c72962d2b4_webhook_storage.py | 1 + .../62addefd9655_webhookstorage_event_ts.py | 2 +- migrations/versions/9d98c1e9c864_vk.py | 8 +++-- social/handlers_discord/base.py | 1 + social/handlers_github/base.py | 1 + social/handlers_telegram/handlers_viribus.py | 1 + social/models/__init__.py | 2 +- social/models/group.py | 10 +++--- social/routes/base.py | 2 +- social/routes/discord.py | 4 +-- social/routes/telegram.py | 4 +-- tests/github_processor.py | 31 +++++-------------- 14 files changed, 43 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 51f4dcc..b9ff4b4 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,12 @@ format: configure source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./social source ./venv/bin/activate && isort ./social source ./venv/bin/activate && black ./social + source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./tests + source ./venv/bin/activate && isort ./tests + source ./venv/bin/activate && black ./tests + source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./migrations + source ./venv/bin/activate && isort ./migrations + source ./venv/bin/activate && black ./migrations configure: venv source ./venv/bin/activate && pip install -r requirements.dev.txt -r requirements.txt diff --git a/migrations/versions/1cacaf803a1d_user_defined_groups.py b/migrations/versions/1cacaf803a1d_user_defined_groups.py index 0c058a9..0475f0d 100644 --- a/migrations/versions/1cacaf803a1d_user_defined_groups.py +++ b/migrations/versions/1cacaf803a1d_user_defined_groups.py @@ -6,9 +6,10 @@ """ -from alembic import op import sqlalchemy as sa -from sqlalchemy.schema import Sequence, CreateSequence +from alembic import op +from sqlalchemy.schema import CreateSequence, Sequence + # revision identifiers, used by Alembic. revision = '1cacaf803a1d' @@ -28,16 +29,18 @@ def upgrade(): sa.Column('create_ts', sa.DateTime(), nullable=False), sa.Column('update_ts', sa.DateTime(), nullable=False), ) - op.execute(''' + op.execute( + ''' INSERT INTO "group" (id, type, is_deleted, last_active_ts, create_ts, update_ts) SELECT id, 'vk_group', False, now(), create_ts, update_ts FROM vk_groups; - ''') + ''' + ) max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "group";')).scalar() op.create_primary_key('group_pk', 'group', ['id']) - op.execute(CreateSequence(Sequence('group_id_seq', max_id+1))) + op.execute(CreateSequence(Sequence('group_id_seq', max_id + 1))) op.alter_column('group', 'id', server_default=sa.text('nextval(\'group_id_seq\')')) op.create_table( @@ -81,7 +84,7 @@ def downgrade(): op.rename_table('vk_group', 'vk_groups') max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "vk_groups";')).scalar() - op.execute(CreateSequence(Sequence('vk_groups_id_seq', max_id+1))) + op.execute(CreateSequence(Sequence('vk_groups_id_seq', max_id + 1))) op.alter_column('vk_groups', 'id', server_default=sa.text('nextval(\'vk_groups_id_seq\')')) op.add_column('vk_groups', sa.Column('create_ts', sa.DateTime())) diff --git a/migrations/versions/57c72962d2b4_webhook_storage.py b/migrations/versions/57c72962d2b4_webhook_storage.py index 3e2be3a..fb22ffe 100644 --- a/migrations/versions/57c72962d2b4_webhook_storage.py +++ b/migrations/versions/57c72962d2b4_webhook_storage.py @@ -5,6 +5,7 @@ Create Date: 2023-03-12 14:22:34.958257 """ + import sqlalchemy as sa from alembic import op diff --git a/migrations/versions/62addefd9655_webhookstorage_event_ts.py b/migrations/versions/62addefd9655_webhookstorage_event_ts.py index 24cdc99..d81ba97 100644 --- a/migrations/versions/62addefd9655_webhookstorage_event_ts.py +++ b/migrations/versions/62addefd9655_webhookstorage_event_ts.py @@ -6,8 +6,8 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. diff --git a/migrations/versions/9d98c1e9c864_vk.py b/migrations/versions/9d98c1e9c864_vk.py index 5b2e9f3..30d873b 100644 --- a/migrations/versions/9d98c1e9c864_vk.py +++ b/migrations/versions/9d98c1e9c864_vk.py @@ -5,8 +5,9 @@ Create Date: 2023-08-19 15:53:19.787309 """ -from alembic import op + import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. @@ -17,14 +18,15 @@ def upgrade(): - op.create_table('vk_groups', + op.create_table( + 'vk_groups', sa.Column('id', sa.Integer(), nullable=False), sa.Column('group_id', sa.Integer(), nullable=False), sa.Column('confirmation_token', sa.String(), nullable=False), sa.Column('secret_key', sa.String(), nullable=False), sa.Column('create_ts', sa.DateTime(), nullable=False), sa.Column('update_ts', sa.DateTime(), nullable=False), - sa.PrimaryKeyConstraint('id') + sa.PrimaryKeyConstraint('id'), ) diff --git a/social/handlers_discord/base.py b/social/handlers_discord/base.py index a6b99bc..59514ac 100644 --- a/social/handlers_discord/base.py +++ b/social/handlers_discord/base.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable + from social.utils.events import EventProcessor diff --git a/social/handlers_github/base.py b/social/handlers_github/base.py index a6b99bc..59514ac 100644 --- a/social/handlers_github/base.py +++ b/social/handlers_github/base.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable + from social.utils.events import EventProcessor diff --git a/social/handlers_telegram/handlers_viribus.py b/social/handlers_telegram/handlers_viribus.py index 8dc74ee..054020b 100644 --- a/social/handlers_telegram/handlers_viribus.py +++ b/social/handlers_telegram/handlers_viribus.py @@ -3,6 +3,7 @@ Для создания нового обработчика создай асинхронную функцию в конце файла с параметрами Update и Context, а потом зарегистрируй ее внутри функции `register_handlers`. """ + import logging from random import choice from string import ascii_letters, digits, punctuation diff --git a/social/models/__init__.py b/social/models/__init__.py index 471545b..f5530c2 100644 --- a/social/models/__init__.py +++ b/social/models/__init__.py @@ -1,4 +1,4 @@ -from .group import TelegramChannel, TelegramChat, VkGroup, VkChat +from .group import TelegramChannel, TelegramChat, VkChat, VkGroup from .webhook_storage import WebhookStorage diff --git a/social/models/group.py b/social/models/group.py index 407fdc6..c1c900e 100644 --- a/social/models/group.py +++ b/social/models/group.py @@ -1,4 +1,4 @@ -from datetime import datetime, UTC +from datetime import UTC, datetime import sqlalchemy as sa from sqlalchemy.orm import Mapped, mapped_column @@ -23,7 +23,7 @@ class Group(Base): class VkGroup(Group): - id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"), primary_key=True) group_id: Mapped[int] confirmation_token: Mapped[str] secret_key: Mapped[str] @@ -34,7 +34,7 @@ class VkGroup(Group): class VkChat(Group): - id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"), primary_key=True) chat_id: Mapped[int] __mapper_args__ = { @@ -43,7 +43,7 @@ class VkChat(Group): class TelegramChannel(Group): - id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"), primary_key=True) channel_id: Mapped[int] __mapper_args__ = { @@ -52,7 +52,7 @@ class TelegramChannel(Group): class TelegramChat(Group): - id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"),primary_key=True) + id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"), primary_key=True) chat_id: Mapped[int] __mapper_args__ = { diff --git a/social/routes/base.py b/social/routes/base.py index 3ed7385..ae79ad2 100644 --- a/social/routes/base.py +++ b/social/routes/base.py @@ -8,10 +8,10 @@ from social.handlers_telegram import get_application as get_telegram from social.settings import get_settings +from .discord import router as discord_router from .github import router as github_router from .telegram import router as telegram_router from .vk import router as vk_router -from .discord import router as discord_router settings = get_settings() diff --git a/social/routes/discord.py b/social/routes/discord.py index 3928418..52c2eca 100644 --- a/social/routes/discord.py +++ b/social/routes/discord.py @@ -1,10 +1,10 @@ import logging -from fastapi import APIRouter, BackgroundTasks, Request, HTTPException +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request from fastapi.responses import JSONResponse from fastapi_sqlalchemy import db -from nacl.signing import VerifyKey from nacl.exceptions import BadSignatureError +from nacl.signing import VerifyKey from social.handlers_discord.base import process_event from social.models.webhook_storage import WebhookStorage, WebhookSystems diff --git a/social/routes/telegram.py b/social/routes/telegram.py index 5146ea0..d3de0df 100644 --- a/social/routes/telegram.py +++ b/social/routes/telegram.py @@ -1,14 +1,14 @@ -from datetime import UTC, datetime import logging from asyncio import create_task +from datetime import UTC, datetime from fastapi import APIRouter, Request from fastapi_sqlalchemy import db from telegram import Update from social.handlers_telegram import get_application -from social.models.webhook_storage import WebhookStorage, WebhookSystems from social.models import TelegramChannel, TelegramChat +from social.models.webhook_storage import WebhookStorage, WebhookSystems from social.settings import get_settings diff --git a/tests/github_processor.py b/tests/github_processor.py index 918b463..97499e6 100644 --- a/tests/github_processor.py +++ b/tests/github_processor.py @@ -13,45 +13,28 @@ def event(): "updated_at": "2023-03-12T18:29:58Z", "has_issues": True, "has_projects": False, - "topics": [ - "auth-service" - ], + "topics": ["auth-service"], "id": 94626404, } def test_str_filter(event: dict): - p = EventProcessor( - dict(action="resolved"), - lambda e: None - ) + p = EventProcessor(dict(action="resolved"), lambda e: None) assert p.check_and_process(event) is True def test_lambda_filter(event: dict): - p = EventProcessor( - { - "created_at": lambda v: datetime.fromisoformat(v).month == 3 - }, - lambda e: None - ) + p = EventProcessor({"created_at": lambda v: datetime.fromisoformat(v).month == 3}, lambda e: None) assert p.check_and_process(event) is True + def test_str_filter_fail(event: dict): - p = EventProcessor( - dict(action="approved"), - lambda e: None - ) + p = EventProcessor(dict(action="approved"), lambda e: None) assert p.check_and_process(event) is False def test_lambda_filter_fail(event: dict): - p = EventProcessor( - { - "created_at": lambda v: datetime.fromisoformat(v).year < 2023 - }, - lambda e: None - ) + p = EventProcessor({"created_at": lambda v: datetime.fromisoformat(v).year < 2023}, lambda e: None) assert p.check_and_process(event) is False @@ -61,6 +44,6 @@ def test_regex_filter(event: dict): "created_at": "2023", "updated_at": r"\d\d\d\d-\d\d-\d\d", }, - lambda e: None + lambda e: None, ) assert p.check_and_process(event) is True From 21188612f675f04826d6cf6bec0bc6b9b921e953 Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 01:40:47 +0300 Subject: [PATCH 11/12] Fix migrations --- migrations/versions/1cacaf803a1d_user_defined_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/versions/1cacaf803a1d_user_defined_groups.py b/migrations/versions/1cacaf803a1d_user_defined_groups.py index 0475f0d..3ca0d70 100644 --- a/migrations/versions/1cacaf803a1d_user_defined_groups.py +++ b/migrations/versions/1cacaf803a1d_user_defined_groups.py @@ -38,7 +38,7 @@ def upgrade(): ''' ) - max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "group";')).scalar() + max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "group";')).scalar() or 0 op.create_primary_key('group_pk', 'group', ['id']) op.execute(CreateSequence(Sequence('group_id_seq', max_id + 1))) op.alter_column('group', 'id', server_default=sa.text('nextval(\'group_id_seq\')')) @@ -83,7 +83,7 @@ def upgrade(): def downgrade(): op.rename_table('vk_group', 'vk_groups') - max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "vk_groups";')).scalar() + max_id = op.get_bind().execute(sa.text('SELECT MAX(id) FROM "vk_groups";')).scalar() or 0 op.execute(CreateSequence(Sequence('vk_groups_id_seq', max_id + 1))) op.alter_column('vk_groups', 'id', server_default=sa.text('nextval(\'vk_groups_id_seq\')')) From 4ec3941f2cd12f2c762060adde80aedebcbdd38c Mon Sep 17 00:00:00 2001 From: Roman Dyakov Date: Mon, 15 Apr 2024 02:10:49 +0300 Subject: [PATCH 12/12] VK --- .../1cacaf803a1d_user_defined_groups.py | 2 +- social/models/group.py | 2 +- social/routes/vk.py | 24 ++++++++++++++++++- social/settings.py | 2 ++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/migrations/versions/1cacaf803a1d_user_defined_groups.py b/migrations/versions/1cacaf803a1d_user_defined_groups.py index 3ca0d70..d506201 100644 --- a/migrations/versions/1cacaf803a1d_user_defined_groups.py +++ b/migrations/versions/1cacaf803a1d_user_defined_groups.py @@ -66,7 +66,7 @@ def upgrade(): op.create_table( 'vk_chat', sa.Column('id', sa.Integer(), nullable=False), - sa.Column('chat_id', sa.Integer(), nullable=False), + sa.Column('peer_id', sa.Integer(), nullable=False), sa.ForeignKeyConstraint( ['id'], ['group.id'], diff --git a/social/models/group.py b/social/models/group.py index c1c900e..1c74850 100644 --- a/social/models/group.py +++ b/social/models/group.py @@ -35,7 +35,7 @@ class VkGroup(Group): class VkChat(Group): id: Mapped[int] = mapped_column(sa.ForeignKey("group.id"), primary_key=True) - chat_id: Mapped[int] + peer_id: Mapped[int] __mapper_args__ = { "polymorphic_identity": "vk_chat", diff --git a/social/routes/vk.py b/social/routes/vk.py index 34e7046..e5e1e81 100644 --- a/social/routes/vk.py +++ b/social/routes/vk.py @@ -1,4 +1,5 @@ import logging +from datetime import UTC, datetime from auth_lib.fastapi import UnionAuth from fastapi import APIRouter, Depends, Request @@ -7,7 +8,7 @@ from pydantic import BaseModel, ConfigDict from social.handlers_telegram import get_application -from social.models.group import VkGroup +from social.models.group import VkChat, VkGroup from social.models.webhook_storage import WebhookStorage, WebhookSystems from social.settings import get_settings from social.utils.string import random_string @@ -54,6 +55,27 @@ async def vk_webhook(request: Request) -> str: ) db.session.commit() + if request_data.get("type") == "message_new": + # Получение сообщения в чате ВК + try: + peer_id = request_data["object"]["message"]["peer_id"] + obj = db.session.query(VkChat).where(VkChat.peer_id == peer_id).one_or_none() + if obj is None: + # Надо будет добавлять название группы + # conversation = requests.post("https://api.vk.com/method/messages.getConversationsById", json={ + # "peer_ids": peer_id, + # "group_id": 222099060, + # "access_token": settings.VK_BOT_TOKEN, + # "v": 5.199, + # }) + # chat_title = conversation["response"]["items"][0]["chat_settings"]["title"] + obj = VkChat(chat_id=peer_id) + db.session.add(obj) + obj.last_active_ts = datetime.now(UTC) + db.session.commit() + except Exception as exc: + logger.exception(exc) + return PlainTextResponse('ok') diff --git a/social/settings.py b/social/settings.py index c272142..f8b2283 100644 --- a/social/settings.py +++ b/social/settings.py @@ -20,6 +20,8 @@ class Settings(BaseSettings): TELEGRAM_BOT_TOKEN: str | None = None + VK_BOT_TOKEN: str | None = None + GITHUB_APP_ID: int | None = None GITHUB_WEBHOOK_SECRET: str | None = None GITHUB_PRIVATE_KEY: str | None = None