restructure & update

.vscode/settings.json

@@ -5,7 +5,7 @@
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.defaultFormatter": "charliermarsh.ruff",
         "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
             "source.organizeImports": "explicit"

Makefile

@@ -59,6 +59,11 @@ docker-build: ## Startup docker with build switch
 docker-build-detached: ## Startup docker with build switch
 	docker-compose up --build -d
 
+docker-restart:
+	docker-compose down
+	docker-compose up --build -d
+
+
 setup: test-setup requirements## setup requirements
 
 setup-detached: test-setup docker-build-detached ## setup & run after downloaded repo detached

docker-compose.yaml

@@ -63,7 +63,7 @@ services:
       - ./mysql/conf.d:/etc/mysql/conf.d
       # - ./mysql/mount:/var/lib/mysql # creates persistence
     ports:
-      - 3306:3306
+      - 3307:3306
     networks:
       - botdetector-network
     healthcheck:
@@ -93,7 +93,7 @@ services:
     # this overrides the env_file for the specific variable
     environment:
       - KAFKA_HOST=kafka:9092
-      - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata
+      - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql/playerdata
       - ENV=DEV
       - POOL_RECYCLE=60
       - POOL_TIMEOUT=30

mysql/docker-entrypoint-initdb.d/01_tables.sql

@@ -83,6 +83,9 @@ CREATE TABLE Predictions (
     vorkath_bot DECIMAL(5, 2) DEFAULT 0,
     barrows_bot DECIMAL(5, 2) DEFAULT 0,
     herblore_bot DECIMAL(5, 2) DEFAULT 0,
+    zulrah_bot DECIMAL(5, 2) DEFAULT 0,
+    gauntlet_bot DECIMAL(5, 2) DEFAULT 0,
+    nex_bot DECIMAL(5, 2) DEFAULT 0,
     unknown_bot DECIMAL(5, 2) DEFAULT 0
 );
 -- Create a table for Feedback

requirements.txt

@@ -1,43 +1,57 @@
-aiokafka==0.8.1
-aiomysql==0.2.0
-annotated-types==0.5.0
-anyio==3.7.1
-async-timeout==4.0.3
-asyncmy==0.2.8
-black==23.9.1
-cfgv==3.4.0
-click==8.1.7
-colorama==0.4.6
-databases==0.8.0
-distlib==0.3.7
-exceptiongroup==1.1.3
-fastapi==0.103.1
-filelock==3.12.4
-greenlet==2.0.2
-h11==0.14.0
-httptools==0.6.0
-identify==2.5.30
-idna==3.4
-kafka-python==2.0.2
-mypy-extensions==1.0.0
-nodeenv==1.8.0
-packaging==23.1
-pathspec==0.11.2
-platformdirs==3.11.0
-pre-commit==3.5.0
-pydantic==2.3.0
-pydantic-settings==2.0.3
-pydantic_core==2.6.3
-PyMySQL==1.1.0
-python-dotenv==1.0.0
-PyYAML==6.0.1
-sniffio==1.3.0
-SQLAlchemy==1.4.49
-starlette==0.27.0
-tomli==2.0.1
-typing_extensions==4.7.1
-uvicorn==0.23.2
-virtualenv==20.24.5
-watchfiles==0.20.0
-websockets==11.0.3
-cryptography==41.0.4
+aiokafka==0.10.0
+aiomysql==0.2.0
+annotated-types==0.6.0
+anyio==4.3.0
+async-timeout==4.0.3
+asyncmy==0.2.9
+attrs==23.1.0
+black==24.3.0
+certifi==2023.7.22
+cffi==1.16.0
+cfgv==3.4.0
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+cryptography==42.0.5
+databases==0.9.0
+distlib==0.3.8
+exceptiongroup==1.2.0
+fastapi==0.110.0
+filelock==3.13.1
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.4
+httptools==0.6.1
+httpx==0.27.0
+hypothesis==6.88.3
+identify==2.5.35
+idna==3.6
+iniconfig==2.0.0
+kafka-python==2.0.2
+mypy-extensions==1.0.0
+nodeenv==1.8.0
+packaging==24.0
+pathspec==0.12.1
+platformdirs==4.2.0
+pluggy==1.3.0
+pre-commit==3.6.2
+pycparser==2.21
+pydantic==2.6.4
+pydantic-settings==2.2.1
+pydantic_core==2.16.3
+PyMySQL==1.1.0
+pytest==7.4.3
+python-dotenv==1.0.1
+PyYAML==6.0.1
+requests==2.31.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+SQLAlchemy==2.0.28
+starlette==0.36.3
+tomli==2.0.1
+typing_extensions==4.10.0
+urllib3==2.0.7
+uvicorn==0.28.0
+virtualenv==20.25.1
+watchfiles==0.21.0
+websockets==12.0

src/api/v2/feedback.py

@@ -2,7 +2,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException, status
 
-from src.app.models.feedback import Feedback
+from src.app.repositories.feedback import Feedback
 from src.app.views.input.feedback import FeedbackInput
 from src.app.views.response.ok import Ok
 from src.core.fastapi.dependencies.session import get_session

src/api/v2/player.py

@@ -23,7 +23,7 @@ async def get_players_kc(
         min_length=1,
         max_length=5,
         description="Name of the player",
-        example=["Player1", "Player2"],
+        examples=["Player1", "Player2"],
     ),
     session=Depends(get_session),
 ):
@@ -49,7 +49,7 @@ async def get_feedback_score(
         min_length=1,
         max_length=5,
         description="Name of the player",
-        example=["Player1", "Player2"],
+        examples=["Player1", "Player2"],
     ),
     session=Depends(get_session),
 ):
@@ -75,7 +75,7 @@ async def get_prediction(
         min_length=1,
         max_length=5,
         description="Name of the player",
-        example=["Player1", "Player2"],
+        examples=["Player1", "Player2"],
     ),
     breakdown: bool = Query(...),
     session=Depends(get_session),

src/api/v2/report.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, status
 from fastapi.exceptions import HTTPException
 
-from src.app.models.report import Report
+from src.app.repositories.report import Report
 from src.app.views.input.report import Detection
 from src.app.views.response.ok import Ok
 

src/app/repositories/player.py

@@ -19,48 +19,39 @@ def __init__(self, session: AsyncSession) -> None:
         self.session = session
 
     async def get_report_score(self, player_names: tuple[str]):
-        """
-        Retrieve Kill Count (KC) data for a list of player names.
+        voter: dbPlayer = aliased(dbPlayer, name="voter")
+        subject: dbPlayer = aliased(dbPlayer, name="subject")
 
-        Args:
-            player_names (list[str]): A list of player names for which KC data is requested.
+        sub_query: Select = select(
+            dbReport.reportedID.distinct().label("reportedID"), dbReport.manual_detect
+        )
+        sub_query = sub_query.join(voter, dbReport.reportingID == voter.id)
+        sub_query = sub_query.where(voter.name.in_(player_names))
+        sub_query = sub_query.where(dbReport.manual_detect == 0)
 
-        Returns:
-            tuple: A tuple of dictionaries containing KC data for each player name. Each dictionary
-                includes the following keys:
-                - "count": The distinct count of reported players.
-                - "possible_ban": Whether the player has a possible ban (True or False).
-                - "confirmed_ban": Whether the player has a confirmed ban (True or False).
-                - "confirmed_player": Whether the player is confirmed as a valid player (True or False).
-                - "manual_detect": Wheter the detection was manual (True or False)
-        """
-        # Create aliases for the tables
-        reporting_player: dbPlayer = aliased(dbPlayer, name="reporting_player")
-        reported_player: dbPlayer = aliased(dbPlayer, name="reported_player")
+        # Create an alias for the subquery
+        sub_query_alias = sub_query.alias("DistinctReports")
 
-        query: Select = select(
-            [
-                func.count(func.distinct(reported_player.id)).label("count"),
-                reported_player.possible_ban,
-                reported_player.confirmed_ban,
-                reported_player.confirmed_player,
-                func.coalesce(dbReport.manual_detect, 0).label("manual_detect"),
-            ]
+        sql: Select = select(
+            func.count(func.distinct(subject.id)).label("count"),
+            subject.possible_ban,
+            subject.confirmed_ban,
+            subject.confirmed_player,
+            func.coalesce(sub_query_alias.c.manual_detect, 0).label("manual_detect"),
         )
-        query = query.select_from(dbReport)
-        query = query.join(
-            reporting_player, dbReport.reportingID == reporting_player.id
-        )
-        query = query.join(reported_player, dbReport.reportedID == reported_player.id)
-        query = query.where(reporting_player.name.in_(player_names))
-        query = query.group_by(
-            reported_player.possible_ban,
-            reported_player.confirmed_ban,
-            reported_player.confirmed_player,
-            dbReport.manual_detect,
+        sql = sql.select_from(sub_query_alias)
+        sql = sql.join(
+            subject, sub_query_alias.c.reportedID == subject.id
+        )  # Use c to access columns
+        sql = sql.group_by(
+            subject.possible_ban,
+            subject.confirmed_ban,
+            subject.confirmed_player,
+            func.coalesce(sub_query_alias.c.manual_detect, 0).label("manual_detect"),
         )
+
         async with self.session:
-            result: AsyncResult = await self.session.execute(query)
+            result: AsyncResult = await self.session.execute(sql)
             await self.session.commit()
         return tuple(result.mappings())
 
@@ -70,12 +61,10 @@ async def get_feedback_score(self, player_names: list[str]):
         fb_subject: dbPlayer = aliased(dbPlayer, name="feedback_subject")
 
         query: Select = select(
-            [
-                func.count(func.distinct(fb_subject.id)).label("count"),
-                fb_subject.possible_ban,
-                fb_subject.confirmed_ban,
-                fb_subject.confirmed_player,
-            ]
+            func.count(func.distinct(fb_subject.id)).label("count"),
+            fb_subject.possible_ban,
+            fb_subject.confirmed_ban,
+            fb_subject.confirmed_player,
         )
         query = query.select_from(dbFeedback)
         query = query.join(fb_voter, dbFeedback.voter_id == fb_voter.id)

src/app/repositories/report.py

@@ -3,7 +3,7 @@
 import time
 
 from src.app.views.input.report import Detection
-from src.core.fastapi.dependencies import kafka
+from src.core import config
 
 logger = logging.getLogger(__name__)
 
@@ -47,6 +47,6 @@ async def parse_data(self, data: list[dict]) -> list[Detection] | None:
     async def send_to_kafka(self, data: list[Detection]) -> None:
         detections = [d.model_dump(mode="json") for d in data]
         await asyncio.gather(
-            *[kafka.report_send_queue.put(detection) for detection in detections]
+            *[config.send_queue.put(detection) for detection in detections]
         )
         return

src/app/views/input/feedback.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, Field, field_validator
 
 
 class FeedbackInput(BaseModel):
@@ -10,31 +10,31 @@ class FeedbackInput(BaseModel):
 
     player_name: str = Field(
         ...,
-        example="Player1",
+        examples=["Player1"],
         min_length=1,
         max_length=50,
         description="Name of the player",
     )
     vote: int = Field(..., ge=-1, le=1, description="Vote is -1, 0 or 1")
     prediction: str = Field(
         ...,
-        example="real_player",
+        examples=["real_player"],
         description="Prediction for the player",
     )
     confidence: Optional[float] = Field(
         0, ge=0, le=1, description="Confidence level of the prediction"
     )
-    subject_id: int = Field(..., example=1, description="ID of the subject")
+    subject_id: int = Field(..., examples=[1], description="ID of the subject")
     feedback_text: Optional[str] = Field(
-        None, example="Test feedback", max_length=250, description="Feedback text"
+        None, examples=["Test feedback"], max_length=250, description="Feedback text"
     )
     proposed_label: Optional[str] = Field(
         None,
-        example="real_player",
+        examples=["real_player"],
         description="Proposed label for the player",
     )
 
-    @validator("player_name")
+    @field_validator("player_name")
     def player_name_validator(cls, value: str):
         match value:
             case _ if 1 <= len(value) <= 12:

src/core/config.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from pydantic_settings import BaseSettings
 
 
@@ -10,3 +12,6 @@ class Settings(BaseSettings):
 
 
 settings = Settings()
+producer = None
+send_queue = None
+sd_event = asyncio.Event()

src/core/database/database.py

@@ -1,6 +1,5 @@
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import declarative_base, sessionmaker
 
 from src.core.config import settings