Post Feedback (#31)

* test case and model change

* define route for post

* wip daily

* min size vs min value

* cleanup anon user

* stuck, daily

* lower case labels

* checkpoint

* checkpoint, add player working

* working valdiator for feedback input

* Update foreign key references in Feedback model

* cleanup of ok debug returns

* checkpoint working post of feedback!

* working code, need to fix tests

* minor fix

* better but not perfect

* bug fix

* change to self.assertions

* add anon users to players, 10

* self.asserts and worked on post feedback valid anon

* Refactor test_post_feedback_valid_anon method to
generate random data

* remove duplicate players and subjects

* remove duplicate players

* remove print

* revert

* valid player names from reports

* Refactor player API tests

* use and_ and first()

* rename class to TestReportAPI

* cleanup debug ouput

* improve logging

* remove unneccesary and

* hardcoded anonymous users

* improve report

---------

Co-authored-by: extreme4all <>

docker-compose.yaml

@@ -79,7 +79,7 @@ services:
       dockerfile: Dockerfile
       target: base
       args:
-        root_path: /
+        root_path: ""
         api_port: 5000
     image: public_api
     # command: bash -c "apt update && apt install -y curl && sleep infinity"

kafka_setup/setup_kafka.py

@@ -34,7 +34,7 @@ def create_topics():
                 replication_factor=1,
             ),
             NewTopic(
-                name="reports",
+                name="report",
                 num_partitions=4,
                 replication_factor=1,
             ),

mysql/docker-entrypoint-initdb.d/02_data.sql

@@ -51,11 +51,10 @@ call InsertRandomPlayers(100, 0,0,1);
 UPDATE Players
 SET
     name = CONCAT('player', id),
-    created_at = NOW() - INTERVAL FLOOR(RAND(42) * 365) DAY,
-    updated_at = NOW() - INTERVAL FLOOR(RAND(41) * 365) DAY,
     normalized_name = CONCAT('player', id)
 ;
 
+
 -- Insert data into the Reports table
 INSERT INTO
     Reports (
@@ -167,7 +166,9 @@ SELECT
         TIMESTAMPDIFF(SECOND, '2020-01-01 00:00:00', '2022-12-31 23:59:59') * RAND(42) 
         + UNIX_TIMESTAMP('2020-01-01 00:00:00')
     )
-FROM `Players` 
+FROM `Players`
+where 1=1
+    AND name not LIKE 'anonymoususer%'
 ORDER BY RAND(42) 
 LIMIT 250
 ;
@@ -235,4 +236,40 @@ UPDATE PredictionsFeedback
     SET proposed_label = prediction
 WHERE 1=1
     AND vote = 1
+;
+
+DELIMITER $$
+
+INSERT INTO Players (
+    name,
+    created_at,
+    updated_at,
+    possible_ban,
+    confirmed_ban,
+    confirmed_player,
+    label_id,
+    label_jagex
+) VALUES 
+    ("anonymoususer 382e728f 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e7259 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e7221 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e71ee 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e71bb 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e7179 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e7133 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e70ef 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e7089 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0),
+    ("anonymoususer 382e6def 87ea 11ee aab6 0242ac120002", NOW(), NOW(), 0, 0, 0, 0, 0)
+;
+
+UPDATE `Players`
+SET
+    created_at = NOW() - INTERVAL FLOOR(RAND(42) * 365) DAY,
+    updated_at = NOW() - INTERVAL FLOOR(RAND(41) * 365) DAY
+;
+UPDATE `Players`
+SET
+    name=replace(name,'-',' '),
+    normalized_name=replace(name,'-',' ')
+WHERE name LIKE 'anonymoususer%'
 ;

src/api/v2/__init__.py

@@ -1,7 +1,8 @@
 from fastapi import APIRouter
 
-from . import player, report
+from . import feedback, player, report
 
 router = APIRouter()
 router.include_router(player.router)
 router.include_router(report.router)
+router.include_router(feedback.router)

src/api/v2/feedback.py

@@ -0,0 +1,28 @@
+import logging
+
+from fastapi import APIRouter, Depends, HTTPException, status
+
+from src.app.models.feedback import Feedback
+from src.app.views.input.feedback import FeedbackInput
+from src.app.views.response.ok import Ok
+from src.core.fastapi.dependencies.session import get_session
+from src.core.fastapi.dependencies.to_jagex_name import to_jagex_name
+
+router = APIRouter(tags=["Feedback"])
+logger = logging.getLogger(__name__)
+
+
+@router.post("/feedback", response_model=Ok, status_code=status.HTTP_201_CREATED)
+async def post_feedback(
+    feedback: FeedbackInput,
+    session=Depends(get_session),
+):
+    """ """
+    _feedback = Feedback(session)
+
+    feedback.player_name = await to_jagex_name(feedback.player_name)
+
+    success, detail = await _feedback.insert_feedback(feedback=feedback)
+    if not success:
+        raise HTTPException(status_code=422, detail=detail)
+    return Ok()

src/api/v2/report.py

@@ -4,14 +4,14 @@
 from src.app.models.report import Report
 from src.app.views.input.report import Detection
 from src.app.views.response.ok import Ok
-from src.core.kafka.report import report_engine
+from src.core.fastapi.dependencies import kafka
 
 router = APIRouter(tags=["Report"])
 
 
 @router.post("/report", status_code=status.HTTP_201_CREATED, response_model=Ok)
 async def post_reports(detection: list[Detection]):
-    report = Report(kafka_engine=report_engine)
+    report = Report()
     data = await report.parse_data(detection)
     if not data:
         raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="invalid data")

src/app/models/feedback.py

@@ -0,0 +1,70 @@
+import logging
+import time
+
+from fastapi.encoders import jsonable_encoder
+from sqlalchemy import and_, func, insert, select
+from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession
+from sqlalchemy.sql.expression import Insert, Select
+
+from src.app.views.input.feedback import FeedbackInput
+from src.core.database.models.feedback import PredictionFeedback as dbFeedback
+from src.core.database.models.player import Player as dbPlayer
+
+logger = logging.getLogger(__name__)
+
+
+class Feedback:
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def insert_feedback(self, feedback: FeedbackInput) -> tuple[bool, str]:
+        sql_select: Select = select(dbPlayer.id)
+        sql_select = sql_select.where(dbPlayer.name == feedback.player_name)
+
+        sql_dupe_check: Select = select(dbFeedback)
+        sql_dupe_check = sql_dupe_check.where(
+            and_(
+                dbFeedback.prediction == feedback.prediction,
+                dbFeedback.subject_id == feedback.subject_id,
+            )
+        )
+
+        sql_insert: Insert = insert(dbFeedback)
+        data = {
+            "voter_id": None,
+            "subject_id": feedback.subject_id,
+            "prediction": feedback.prediction,
+            "confidence": feedback.confidence,
+            "vote": feedback.vote,
+            "feedback_text": feedback.feedback_text,
+            "proposed_label": feedback.proposed_label,
+        }
+
+        async with self.session:
+            result: AsyncResult = await self.session.execute(sql_select)
+            result = result.first()
+
+            # check if voter exists
+            if not result:
+                logger.info({"voter_does_not_exist": FeedbackInput})
+                await self.session.rollback()
+                return False, "voter_does_not_exist"
+
+            voter_id = result["id"]
+            sql_dupe_check = sql_dupe_check.where(dbFeedback.voter_id == voter_id)
+
+            result: AsyncResult = await self.session.execute(sql_dupe_check)
+            result = result.first()
+
+            # check if duplicate record
+            if result:
+                logger.info({"duplicate_record": FeedbackInput, "voter id": voter_id})
+                await self.session.rollback()
+                return False, "duplicate_record"
+
+            # add voter_id and insert
+            data["voter_id"] = voter_id
+            sql_insert = sql_insert.values(data)
+            result: AsyncResult = await self.session.execute(sql_insert)
+            await self.session.commit()
+        return True, "success"

src/app/models/player.py

@@ -1,14 +1,22 @@
+import logging
+import time
+
 from fastapi.encoders import jsonable_encoder
 from sqlalchemy import func, select
 from sqlalchemy.engine import Result
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession
 from sqlalchemy.orm import aliased
 from sqlalchemy.sql.expression import Select
 
+from src.app.views.input.feedback import FeedbackInput
 from src.core.database.models.feedback import PredictionFeedback as dbFeedback
 from src.core.database.models.player import Player as dbPlayer
 from src.core.database.models.prediction import Prediction as dbPrediction
 from src.core.database.models.report import Report as dbReport
+from src.core.fastapi.dependencies.to_jagex_name import to_jagex_name
+
+logger = logging.getLogger(__name__)
 
 
 class Player:

src/app/models/report.py

@@ -1,15 +1,15 @@
+import asyncio
 import logging
 import time
 
 from src.app.views.input.report import Detection
-from src.core.kafka.engine import AioKafkaEngine
+from src.core.fastapi.dependencies import kafka
 
 logger = logging.getLogger(__name__)
 
 
 class Report:
-    def __init__(self, kafka_engine: AioKafkaEngine) -> None:
-        self.kafka_engine = kafka_engine
+    def __init__(self) -> None:
         pass
 
     def _check_data_size(self, data: list[Detection]) -> list[Detection] | None:
@@ -45,7 +45,8 @@ async def parse_data(self, data: list[dict]) -> list[Detection] | None:
         return data
 
     async def send_to_kafka(self, data: list[Detection]) -> None:
-        for detection in data:
-            detection = detection.model_dump_json()
-            self.kafka_engine.message_queue.put_nowait(detection)
+        detections = [d.model_dump(mode="json") for d in data]
+        await asyncio.gather(
+            *[kafka.report_send_queue.put(detection) for detection in detections]
+        )
         return

src/app/views/input/feedback.py

@@ -1,27 +1,24 @@
-import re
-from typing import List, Optional
+from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, constr, validator
 
 
 class FeedbackInput(BaseModel):
     """
     Class representing prediction feedback input.
     """
 
-    player_name: str = Field(
+    player_name: constr(strip_whitespace=True) = Field(
         ...,
         example="Player1",
         min_length=1,
-        max_length=13,
+        max_length=50,
         description="Name of the player",
     )
     vote: int = Field(..., ge=-1, le=1, description="Vote is -1, 0 or 1")
     prediction: str = Field(
         ...,
-        example="Real_Player",
-        min_length=1,
-        max_length=13,
+        example="real_player",
         description="Prediction for the player",
     )
     confidence: Optional[float] = Field(
@@ -35,31 +32,14 @@ class FeedbackInput(BaseModel):
         None,
         example="real_player",
         description="Proposed label for the player",
-        enum=[
-            "real_player",
-            "pvm_melee_bot",
-            "smithing_bot",
-            "magic_bot",
-            "fishing_bot",
-            "mining_bot",
-            "crafting_bot",
-            "pvm_ranged_magic_bot",
-            "pvm_ranged_bot",
-            "hunter_bot",
-            "fletching_bot",
-            "clue_scroll_bot",
-            "lms_bot",
-            "agility_bot",
-            "wintertodt_bot",
-            "runecrafting_bot",
-            "zalcano_bot",
-            "woodcutting_bot",
-            "thieving_bot",
-            "soul_wars_bot",
-            "cooking_bot",
-            "vorkath_bot",
-            "barrows_bot",
-            "herblore_bot",
-            "unknown_bot",
-        ],
     )
+
+    @validator("player_name")
+    def uuid_format(cls, value: str):
+        match value:
+            case _ if 1 <= len(value) <= 12:
+                return value
+            case _ if value.lower().startswith("anonymoususer"):
+                return value
+            case _:
+                raise ValueError("Invalid format for player_name")

src/core/database/models/feedback.py

@@ -17,8 +17,8 @@ class PredictionFeedback(Base):
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     ts = Column(TIMESTAMP, nullable=False, server_default="CURRENT_TIMESTAMP")
-    voter_id = Column(Integer, ForeignKey("FK_Voter_ID"), nullable=False)
-    subject_id = Column(Integer, ForeignKey("FK_Subject_ID"), nullable=False)
+    voter_id = Column(Integer, ForeignKey("Players.id"), nullable=False)
+    subject_id = Column(Integer, ForeignKey("Players.id"), nullable=False)
     prediction = Column(String(50), nullable=False)
     confidence = Column(Float, nullable=False)
     vote = Column(Integer, nullable=False, server_default="0")