Merge pull request #51 from Bot-detector/api_v3

Api v3

.env-example

@@ -1,3 +1,4 @@
 token = verify_ban
-api = http://localhost:5000
-secret_token = super_secret_token
+detector_api = http://localhost:5000
+secret_token = super_secret_token
+private_api = http://localhost:5001/v3

Makefile

@@ -0,0 +1,8 @@
+docker-restart:
+	docker compose down
+	docker compose up --build -d
+
+docker-test:
+	docker compose down
+	docker compose up --build -d
+	pytest

api/MachineLearning/classifier.py

@@ -6,16 +6,21 @@
 import numpy as np
 import pandas as pd
 from sklearn.ensemble import RandomForestClassifier
-from sklearn.metrics import (balanced_accuracy_score, classification_report,
-                             roc_auc_score)
+from sklearn.metrics import (
+    balanced_accuracy_score,
+    classification_report,
+    roc_auc_score,
+)
 
 logger = logging.getLogger(__name__)
 
+
 class classifier(RandomForestClassifier):
     """
     This class is a wrapper for RandomForestClassifier.
     It adds the ability to save and load the model.
     """
+
     working_directory = os.path.dirname(os.path.realpath(__file__))
     path = os.path.join(working_directory, "models")
     if not os.path.exists(path):
@@ -56,7 +61,6 @@ def __best_file_path(self, startwith: str):
                 # add dict to array
                 files.append(d)
 
-
         if not files:
             return None
 
@@ -96,7 +100,6 @@ def save(self):
             compress=3,
         )
 
-
     def score(self, test_y, test_x):
         """
         Calculate the accuracy and roc_auc score for the classifier.
@@ -121,4 +124,4 @@ def score(self, test_y, test_x):
         labels = ["Not bot", "bot"] if len(labels) == 2 else labels
 
         logger.info(classification_report(test_y, pred_y, target_names=labels))
-        return self.accuracy, self.roc_auc
+        return self.accuracy, self.roc_auc

api/app.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from datetime import date
 from typing import List
 
 import pandas as pd
@@ -11,7 +12,6 @@
 from api.cogs import predict
 from api.cogs import requests as req
 from api.MachineLearning import classifier, data
-from datetime import date
 
 app = config.app
 
@@ -47,7 +47,7 @@ async def root():
     """
     This endpoint is used to check if the api is running.
     """
-    return {"detail": "hello worldz"}
+    return {"detail": "hello world"}
 
 
 @app.get("/startup")

api/cogs/predict.py

@@ -1,12 +1,13 @@
+import logging
 import time
 from typing import List
 
 import numpy as np
 import pandas as pd
+
 from api import config
 from api.MachineLearning import data
 from api.MachineLearning.classifier import classifier
-import logging
 
 logger = logging.getLogger(__name__)
 

api/cogs/requests.py

@@ -1,7 +1,9 @@
+import asyncio
 import logging
-import api.config as config
+
 import aiohttp
-import asyncio
+
+import api.config as config
 
 logger = logging.getLogger(__name__)
 
@@ -13,17 +15,14 @@ async def make_request(url: str, params: dict, headers: dict = {}) -> list[dict]
     _secure_params["token"] = "***"
 
     # Log the URL and secure parameters for debugging
-    logger.info({"url": url.split("/v")[-1], "params": _secure_params})
+    logger.info({"url": f"v{url.split('/v')[-1]}", "params": _secure_params})
 
     # Use aiohttp to make an asynchronous GET request
     async with aiohttp.ClientSession() as session:
         async with session.get(url=url, params=params, headers=headers) as resp:
             # Check if the response status is OK (200)
             if not resp.ok:
-                error_message = (
-                    f"response status {resp.status} "
-                    f"response body: {await resp.text()}"
-                )
+                error_message = {"status": resp.status, "body": await resp.text()}
                 # Log the error message and raise a ValueError
                 logger.error(error_message)
                 raise ValueError(error_message)
@@ -52,8 +51,8 @@ async def retry_request(url: str, params: dict) -> list[dict]:
             _secure_params = params.copy()
             _secure_params["token"] = "***"
             logger.error({"url": url, "params": _secure_params, "error": str(e)})
-            await asyncio.sleep(15)
-            retry += 1
+        await asyncio.sleep(15)
+        retry += 1
 
 
 # Define an asynchronous function to get labels from an API
@@ -70,7 +69,7 @@ async def get_labels():
 
 
 async def get_player_data(label_id: int, limit: int = 5000):
-    url = "http://private-api-svc.bd-prd.svc:5000/v2/player"
+    url = f"{config.private_api}/v2/player"
 
     params = {
         "player_id": 1,
@@ -100,7 +99,7 @@ async def get_player_data(label_id: int, limit: int = 5000):
 
 
 async def get_hiscore_data(label_id: int, limit: int = 5000):
-    url = "http://private-api-svc.bd-prd.svc:5000/v2/highscore/latest"  # TODO: fix hardcoded
+    url = f"{config.private_api}/v2/highscore/latest"
     params = {"player_id": 1, "label_id": label_id, "many": 1, "limit": limit}
 
     # Initialize a list to store hiscore data
@@ -124,15 +123,15 @@ async def get_hiscore_data(label_id: int, limit: int = 5000):
 
 
 async def get_prediction_data(player_id: int = 0, limit: int = 0):
-    url = "http://private-api-svc.bd-prd.svc:5000/v2/highscore/latest"  # TODO: fix hardcoded
+    url = f"{config.private_api}/v2/highscore/latest"
     params = {"player_id": player_id, "many": 1, "limit": limit}
 
     data = await retry_request(url=url, params=params)
     return data
 
 
 async def post_prediction(data: list[dict]):
-    url = f"{config.detector_api}/v1/prediction"
+    url = f"{config.detector_api}/prediction"
     params = {"token": config.token}
 
     while True:

api/config.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import sys
@@ -9,32 +10,47 @@
 load_dotenv(find_dotenv(), verbose=True)
 
 # get env variables
+# TODO: convert to pydantid_settings
 token = os.environ.get("token")
-detector_api = os.environ.get("api")
+detector_api = os.environ.get("detector_api")
 secret_token = os.environ.get("secret_token")
+private_api = os.environ.get("private_api")
 
+assert token is not None
+assert detector_api is not None
+assert secret_token is not None
+assert private_api is not None
+
+# TODO: move to app.py // rename that to server.py
 app = FastAPI()
 
+# TODO: move to logging_config.py
 # setup logging
-logger = logging.getLogger()
-file_handler = logging.FileHandler(filename="error.log", mode="a")
-stream_handler = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter(
+    json.dumps(
+        {
+            "ts": "%(asctime)s",
+            "name": "%(name)s",
+            "function": "%(funcName)s",
+            "level": "%(levelname)s",
+            "msg": json.dumps("%(message)s"),
+        }
+    )
+)
 
-logging.basicConfig(filename="error.log", level=logging.DEBUG)
+stream_handler = logging.StreamHandler(sys.stdout)
 
-# log formatting
-formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-file_handler.setFormatter(formatter)
 stream_handler.setFormatter(formatter)
 
-# add handler
-logger.addHandler(file_handler)
-logger.addHandler(stream_handler)
+handlers = [stream_handler]
+
+logging.basicConfig(level=logging.DEBUG, handlers=handlers)
+
 
-logging.getLogger("requests").setLevel(logging.DEBUG)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-logging.getLogger("uvicorn").setLevel(logging.DEBUG)
-logging.getLogger("uvicorn.error").propagate = False
+# logging.getLogger("requests").setLevel(logging.DEBUG)
+# logging.getLogger("urllib3").setLevel(logging.WARNING)
+# logging.getLogger("uvicorn").setLevel(logging.DEBUG)
+# logging.getLogger("uvicorn.error").propagate = False
 
 BATCH_AMOUNT = 5_000
 

docker-compose.yml

@@ -1,62 +1,99 @@
-version: '3'
 services:
   mysql:
+    container_name: database
     build:
-      context: ../bot-detector-mysql
-      dockerfile: Dockerfile
-    image: bot-detector/bd-mysql:latest
+      context: ./mysql
+    image: bot-detector/mysql:latest
     environment:
       - MYSQL_ROOT_PASSWORD=root_bot_buster
-      - MYSQL_USER=botssuck
-      - MYSQL_PASSWORD=botdetector
     volumes:
-      - '../bot-detector-mysql/mount:/var/lib/mysql'
-      - '../bot-detector-mysql/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/'
+      - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+      # - ./mysql/mount:/var/lib/mysql # creates persistence
     ports:
-      - "3306:3306"
+      - 3307:3306
     networks:
       - botdetector-network
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 20s
+      retries: 10
 
-  api:
+  mysql_setup:
+    container_name: mysql_setup
+    image: bot-detector/mysql_setup
     build:
-      context: ../Bot-Detector-Core-Files
-      dockerfile: Dockerfile
-      args:
-        root_path: ''
-        api_port: 5000
-    image: bot-detector/bd-api:latest
+      context: ./mysql_setup
+    command: ["python", "-u","setup_mysql.py"]
+    networks:
+      - botdetector-network
+    depends_on:
+      mysql:
+        condition: service_healthy
+
+  core_api:
+    image: quay.io/bot_detector/bd-core-files:4f82016
+    command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/*
     environment:
       - sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata
       - discord_sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/discord
       - token=verify_ban
-    volumes:
-      - '../Bot-Detector-Core-Files/api:/code/api'
+      - env=DEV
+    # Ports exposed to the other services but not to the host machine
+    expose:
+      - 5000
+    ports:
+      - 5001:5000
+    networks:
+      - botdetector-network
+    depends_on:
+      mysql_setup:
+        condition: service_completed_successfully
+
+  private_api:
+    image: quay.io/bot_detector/private-api:e96c31a
+    command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/*
+    # Ports exposed to the other services but not to the host machine
+    expose: 
+      - 5000
     ports:
-      - "5000:5000"
+      - 5002:5000
     networks:
       - botdetector-network
+    # this overrides the env_file for the specific variable
+    environment:
+      - KAFKA_HOST=kafka:9092
+      - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata
+      - ENV=DEV
+      - POOL_RECYCLE=60
+      - POOL_TIMEOUT=30
     depends_on:
-      - mysql
-  machine-learning:
+      mysql_setup:
+        condition: service_completed_successfully
+
+  machine_learning:
+    container_name: bd-ml
     build:
       context: .
       dockerfile: Dockerfile
       target: base
       args:
-        root_path: '/'
+        root_path: /
         api_port: 8000
-    container_name: bd-ml
     command: uvicorn api.app:app --host 0.0.0.0 --reload --reload-include api/*
-    env_file:
-      - .env
+    environment:
+      - token=verify_ban
+      - secret_token=super_secret_token
+      - private_api=http://private_api:5000
+      - detector_api=http://core_api:5000
     volumes:
-      - ../bot-detector-ML/api:/project/api
+      - ./api:/project/api
     ports:
-      - 8000:8000
+      - 5003:8000
     networks:
       - botdetector-network
     depends_on:
-      - api
+      - private_api
+      - core_api
 
 networks:
   botdetector-network:

mysql/Dockerfile

@@ -0,0 +1,3 @@
+FROM mysql:latest
+
+EXPOSE 3306

mysql/docker-entrypoint-initdb.d/00_init.sql

@@ -0,0 +1 @@
+CREATE DATABASE playerdata;