diff --git a/.env-example b/.env-example index af0f793..ef49a87 100644 --- a/.env-example +++ b/.env-example @@ -1,3 +1,4 @@ token = verify_ban -api = http://localhost:5000 -secret_token = super_secret_token \ No newline at end of file +detector_api = http://localhost:5000 +secret_token = super_secret_token +private_api = http://localhost:5001/v3 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..98b6927 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +docker-restart: + docker compose down + docker compose up --build -d + +docker-test: + docker compose down + docker compose up --build -d + pytest \ No newline at end of file diff --git a/api/MachineLearning/classifier.py b/api/MachineLearning/classifier.py index 9df7a82..143e64b 100644 --- a/api/MachineLearning/classifier.py +++ b/api/MachineLearning/classifier.py @@ -6,16 +6,21 @@ import numpy as np import pandas as pd from sklearn.ensemble import RandomForestClassifier -from sklearn.metrics import (balanced_accuracy_score, classification_report, - roc_auc_score) +from sklearn.metrics import ( + balanced_accuracy_score, + classification_report, + roc_auc_score, +) logger = logging.getLogger(__name__) + class classifier(RandomForestClassifier): """ This class is a wrapper for RandomForestClassifier. It adds the ability to save and load the model. """ + working_directory = os.path.dirname(os.path.realpath(__file__)) path = os.path.join(working_directory, "models") if not os.path.exists(path): @@ -56,7 +61,6 @@ def __best_file_path(self, startwith: str): # add dict to array files.append(d) - if not files: return None @@ -96,7 +100,6 @@ def save(self): compress=3, ) - def score(self, test_y, test_x): """ Calculate the accuracy and roc_auc score for the classifier. @@ -121,4 +124,4 @@ def score(self, test_y, test_x): labels = ["Not bot", "bot"] if len(labels) == 2 else labels logger.info(classification_report(test_y, pred_y, target_names=labels)) - return self.accuracy, self.roc_auc \ No newline at end of file + return self.accuracy, self.roc_auc diff --git a/api/app.py b/api/app.py index 307cdc3..1e23477 100644 --- a/api/app.py +++ b/api/app.py @@ -1,5 +1,6 @@ import asyncio import logging +from datetime import date from typing import List import pandas as pd @@ -11,7 +12,6 @@ from api.cogs import predict from api.cogs import requests as req from api.MachineLearning import classifier, data -from datetime import date app = config.app @@ -47,7 +47,7 @@ async def root(): """ This endpoint is used to check if the api is running. """ - return {"detail": "hello worldz"} + return {"detail": "hello world"} @app.get("/startup") diff --git a/api/cogs/predict.py b/api/cogs/predict.py index f7e8117..3a18300 100644 --- a/api/cogs/predict.py +++ b/api/cogs/predict.py @@ -1,12 +1,13 @@ +import logging import time from typing import List import numpy as np import pandas as pd + from api import config from api.MachineLearning import data from api.MachineLearning.classifier import classifier -import logging logger = logging.getLogger(__name__) diff --git a/api/cogs/requests.py b/api/cogs/requests.py index b8e7d97..9104d2e 100644 --- a/api/cogs/requests.py +++ b/api/cogs/requests.py @@ -1,7 +1,9 @@ +import asyncio import logging -import api.config as config + import aiohttp -import asyncio + +import api.config as config logger = logging.getLogger(__name__) @@ -13,17 +15,14 @@ async def make_request(url: str, params: dict, headers: dict = {}) -> list[dict] _secure_params["token"] = "***" # Log the URL and secure parameters for debugging - logger.info({"url": url.split("/v")[-1], "params": _secure_params}) + logger.info({"url": f"v{url.split('/v')[-1]}", "params": _secure_params}) # Use aiohttp to make an asynchronous GET request async with aiohttp.ClientSession() as session: async with session.get(url=url, params=params, headers=headers) as resp: # Check if the response status is OK (200) if not resp.ok: - error_message = ( - f"response status {resp.status} " - f"response body: {await resp.text()}" - ) + error_message = {"status": resp.status, "body": await resp.text()} # Log the error message and raise a ValueError logger.error(error_message) raise ValueError(error_message) @@ -52,8 +51,8 @@ async def retry_request(url: str, params: dict) -> list[dict]: _secure_params = params.copy() _secure_params["token"] = "***" logger.error({"url": url, "params": _secure_params, "error": str(e)}) - await asyncio.sleep(15) - retry += 1 + await asyncio.sleep(15) + retry += 1 # Define an asynchronous function to get labels from an API @@ -70,7 +69,7 @@ async def get_labels(): async def get_player_data(label_id: int, limit: int = 5000): - url = "http://private-api-svc.bd-prd.svc:5000/v2/player" + url = f"{config.private_api}/v2/player" params = { "player_id": 1, @@ -100,7 +99,7 @@ async def get_player_data(label_id: int, limit: int = 5000): async def get_hiscore_data(label_id: int, limit: int = 5000): - url = "http://private-api-svc.bd-prd.svc:5000/v2/highscore/latest" # TODO: fix hardcoded + url = f"{config.private_api}/v2/highscore/latest" params = {"player_id": 1, "label_id": label_id, "many": 1, "limit": limit} # Initialize a list to store hiscore data @@ -124,7 +123,7 @@ async def get_hiscore_data(label_id: int, limit: int = 5000): async def get_prediction_data(player_id: int = 0, limit: int = 0): - url = "http://private-api-svc.bd-prd.svc:5000/v2/highscore/latest" # TODO: fix hardcoded + url = f"{config.private_api}/v2/highscore/latest" params = {"player_id": player_id, "many": 1, "limit": limit} data = await retry_request(url=url, params=params) @@ -132,7 +131,7 @@ async def get_prediction_data(player_id: int = 0, limit: int = 0): async def post_prediction(data: list[dict]): - url = f"{config.detector_api}/v1/prediction" + url = f"{config.detector_api}/prediction" params = {"token": config.token} while True: diff --git a/api/config.py b/api/config.py index 6f80729..c3eb01e 100644 --- a/api/config.py +++ b/api/config.py @@ -1,3 +1,4 @@ +import json import logging import os import sys @@ -9,32 +10,47 @@ load_dotenv(find_dotenv(), verbose=True) # get env variables +# TODO: convert to pydantid_settings token = os.environ.get("token") -detector_api = os.environ.get("api") +detector_api = os.environ.get("detector_api") secret_token = os.environ.get("secret_token") +private_api = os.environ.get("private_api") +assert token is not None +assert detector_api is not None +assert secret_token is not None +assert private_api is not None + +# TODO: move to app.py // rename that to server.py app = FastAPI() +# TODO: move to logging_config.py # setup logging -logger = logging.getLogger() -file_handler = logging.FileHandler(filename="error.log", mode="a") -stream_handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter( + json.dumps( + { + "ts": "%(asctime)s", + "name": "%(name)s", + "function": "%(funcName)s", + "level": "%(levelname)s", + "msg": json.dumps("%(message)s"), + } + ) +) -logging.basicConfig(filename="error.log", level=logging.DEBUG) +stream_handler = logging.StreamHandler(sys.stdout) -# log formatting -formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") -file_handler.setFormatter(formatter) stream_handler.setFormatter(formatter) -# add handler -logger.addHandler(file_handler) -logger.addHandler(stream_handler) +handlers = [stream_handler] + +logging.basicConfig(level=logging.DEBUG, handlers=handlers) + -logging.getLogger("requests").setLevel(logging.DEBUG) -logging.getLogger("urllib3").setLevel(logging.WARNING) -logging.getLogger("uvicorn").setLevel(logging.DEBUG) -logging.getLogger("uvicorn.error").propagate = False +# logging.getLogger("requests").setLevel(logging.DEBUG) +# logging.getLogger("urllib3").setLevel(logging.WARNING) +# logging.getLogger("uvicorn").setLevel(logging.DEBUG) +# logging.getLogger("uvicorn.error").propagate = False BATCH_AMOUNT = 5_000 diff --git a/docker-compose.yml b/docker-compose.yml index df38df5..cc61bae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,99 @@ -version: '3' services: mysql: + container_name: database build: - context: ../bot-detector-mysql - dockerfile: Dockerfile - image: bot-detector/bd-mysql:latest + context: ./mysql + image: bot-detector/mysql:latest environment: - MYSQL_ROOT_PASSWORD=root_bot_buster - - MYSQL_USER=botssuck - - MYSQL_PASSWORD=botdetector volumes: - - '../bot-detector-mysql/mount:/var/lib/mysql' - - '../bot-detector-mysql/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/' + - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + # - ./mysql/mount:/var/lib/mysql # creates persistence ports: - - "3306:3306" + - 3307:3306 networks: - botdetector-network + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 20s + retries: 10 - api: + mysql_setup: + container_name: mysql_setup + image: bot-detector/mysql_setup build: - context: ../Bot-Detector-Core-Files - dockerfile: Dockerfile - args: - root_path: '' - api_port: 5000 - image: bot-detector/bd-api:latest + context: ./mysql_setup + command: ["python", "-u","setup_mysql.py"] + networks: + - botdetector-network + depends_on: + mysql: + condition: service_healthy + + core_api: + image: quay.io/bot_detector/bd-core-files:4f82016 + command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* environment: - sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/playerdata - discord_sql_uri=mysql+asyncmy://root:root_bot_buster@mysql:3306/discord - token=verify_ban - volumes: - - '../Bot-Detector-Core-Files/api:/code/api' + - env=DEV + # Ports exposed to the other services but not to the host machine + expose: + - 5000 + ports: + - 5001:5000 + networks: + - botdetector-network + depends_on: + mysql_setup: + condition: service_completed_successfully + + private_api: + image: quay.io/bot_detector/private-api:e96c31a + command: uvicorn src.core.server:app --host 0.0.0.0 --reload --reload-include src/* + # Ports exposed to the other services but not to the host machine + expose: + - 5000 ports: - - "5000:5000" + - 5002:5000 networks: - botdetector-network + # this overrides the env_file for the specific variable + environment: + - KAFKA_HOST=kafka:9092 + - DATABASE_URL=mysql+aiomysql://root:root_bot_buster@mysql:3306/playerdata + - ENV=DEV + - POOL_RECYCLE=60 + - POOL_TIMEOUT=30 depends_on: - - mysql - machine-learning: + mysql_setup: + condition: service_completed_successfully + + machine_learning: + container_name: bd-ml build: context: . dockerfile: Dockerfile target: base args: - root_path: '/' + root_path: / api_port: 8000 - container_name: bd-ml command: uvicorn api.app:app --host 0.0.0.0 --reload --reload-include api/* - env_file: - - .env + environment: + - token=verify_ban + - secret_token=super_secret_token + - private_api=http://private_api:5000 + - detector_api=http://core_api:5000 volumes: - - ../bot-detector-ML/api:/project/api + - ./api:/project/api ports: - - 8000:8000 + - 5003:8000 networks: - botdetector-network depends_on: - - api + - private_api + - core_api networks: botdetector-network: diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..659f748 --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,3 @@ +FROM mysql:latest + +EXPOSE 3306 \ No newline at end of file diff --git a/mysql/docker-entrypoint-initdb.d/00_init.sql b/mysql/docker-entrypoint-initdb.d/00_init.sql new file mode 100644 index 0000000..7d82066 --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/00_init.sql @@ -0,0 +1 @@ +CREATE DATABASE playerdata; \ No newline at end of file diff --git a/mysql/docker-entrypoint-initdb.d/01_tables.sql b/mysql/docker-entrypoint-initdb.d/01_tables.sql new file mode 100644 index 0000000..8d49108 --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/01_tables.sql @@ -0,0 +1,546 @@ +USE playerdata; + +CREATE TABLE Players ( + id INT PRIMARY KEY AUTO_INCREMENT, + name TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + possible_ban BOOLEAN, + confirmed_ban BOOLEAN, + confirmed_player BOOLEAN, + label_id INTEGER, + label_jagex INTEGER, + ironman BOOLEAN, + hardcore_ironman BOOLEAN, + ultimate_ironman BOOLEAN, + normalized_name TEXT +); + +CREATE TABLE Reports ( + ID BIGINT PRIMARY KEY AUTO_INCREMENT, + created_at TIMESTAMP, + reportedID INT, + reportingID INT, + region_id INT, + x_coord INT, + y_coord INT, + z_coord INT, + timestamp TIMESTAMP, + manual_detect SMALLINT, + on_members_world INT, + on_pvp_world SMALLINT, + world_number INT, + equip_head_id INT, + equip_amulet_id INT, + equip_torso_id INT, + equip_legs_id INT, + equip_boots_id INT, + equip_cape_id INT, + equip_hands_id INT, + equip_weapon_id INT, + equip_shield_id INT, + equip_ge_value BIGINT, + CONSTRAINT FK_Reported_Players_id FOREIGN KEY (reportedID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT FK_Reporting_Players_id FOREIGN KEY (reportingID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + + +CREATE TABLE Predictions ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(12), + prediction VARCHAR(50), + created TIMESTAMP, + predicted_confidence DECIMAL(5, 2), + real_player DECIMAL(5, 2) DEFAULT 0, + pvm_melee_bot DECIMAL(5, 2) DEFAULT 0, + smithing_bot DECIMAL(5, 2) DEFAULT 0, + magic_bot DECIMAL(5, 2) DEFAULT 0, + fishing_bot DECIMAL(5, 2) DEFAULT 0, + mining_bot DECIMAL(5, 2) DEFAULT 0, + crafting_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_magic_bot DECIMAL(5, 2) DEFAULT 0, + pvm_ranged_bot DECIMAL(5, 2) DEFAULT 0, + hunter_bot DECIMAL(5, 2) DEFAULT 0, + fletching_bot DECIMAL(5, 2) DEFAULT 0, + clue_scroll_bot DECIMAL(5, 2) DEFAULT 0, + lms_bot DECIMAL(5, 2) DEFAULT 0, + agility_bot DECIMAL(5, 2) DEFAULT 0, + wintertodt_bot DECIMAL(5, 2) DEFAULT 0, + runecrafting_bot DECIMAL(5, 2) DEFAULT 0, + zalcano_bot DECIMAL(5, 2) DEFAULT 0, + woodcutting_bot DECIMAL(5, 2) DEFAULT 0, + thieving_bot DECIMAL(5, 2) DEFAULT 0, + soul_wars_bot DECIMAL(5, 2) DEFAULT 0, + cooking_bot DECIMAL(5, 2) DEFAULT 0, + vorkath_bot DECIMAL(5, 2) DEFAULT 0, + barrows_bot DECIMAL(5, 2) DEFAULT 0, + herblore_bot DECIMAL(5, 2) DEFAULT 0, + unknown_bot DECIMAL(5, 2) DEFAULT 0 +); + +CREATE TABLE playerHiscoreData ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT '0', + attack int DEFAULT '0', + defence int DEFAULT '0', + strength int DEFAULT '0', + hitpoints int DEFAULT '0', + ranged int DEFAULT '0', + prayer int DEFAULT '0', + magic int DEFAULT '0', + cooking int DEFAULT '0', + woodcutting int DEFAULT '0', + fletching int DEFAULT '0', + fishing int DEFAULT '0', + firemaking int DEFAULT '0', + crafting int DEFAULT '0', + smithing int DEFAULT '0', + mining int DEFAULT '0', + herblore int DEFAULT '0', + agility int DEFAULT '0', + thieving int DEFAULT '0', + slayer int DEFAULT '0', + farming int DEFAULT '0', + runecraft int DEFAULT '0', + hunter int DEFAULT '0', + construction int DEFAULT '0', + league int DEFAULT '0', + bounty_hunter_hunter int DEFAULT '0', + bounty_hunter_rogue int DEFAULT '0', + cs_all int DEFAULT '0', + cs_beginner int DEFAULT '0', + cs_easy int DEFAULT '0', + cs_medium int DEFAULT '0', + cs_hard int DEFAULT '0', + cs_elite int DEFAULT '0', + cs_master int DEFAULT '0', + lms_rank int DEFAULT '0', + soul_wars_zeal int DEFAULT '0', + abyssal_sire int DEFAULT '0', + alchemical_hydra int DEFAULT '0', + barrows_chests int DEFAULT '0', + bryophyta int DEFAULT '0', + callisto int DEFAULT '0', + cerberus int DEFAULT '0', + chambers_of_xeric int DEFAULT '0', + chambers_of_xeric_challenge_mode int DEFAULT '0', + chaos_elemental int DEFAULT '0', + chaos_fanatic int DEFAULT '0', + commander_zilyana int DEFAULT '0', + corporeal_beast int DEFAULT '0', + crazy_archaeologist int DEFAULT '0', + dagannoth_prime int DEFAULT '0', + dagannoth_rex int DEFAULT '0', + dagannoth_supreme int DEFAULT '0', + deranged_archaeologist int DEFAULT '0', + general_graardor int DEFAULT '0', + giant_mole int DEFAULT '0', + grotesque_guardians int DEFAULT '0', + hespori int DEFAULT '0', + kalphite_queen int DEFAULT '0', + king_black_dragon int DEFAULT '0', + kraken int DEFAULT '0', + kreearra int DEFAULT '0', + kril_tsutsaroth int DEFAULT '0', + mimic int DEFAULT '0', + nex int DEFAULT '0', + nightmare int DEFAULT '0', + phosanis_nightmare int DEFAULT '0', + obor int DEFAULT '0', + phantom_muspah int DEFAULT '0', + sarachnis int DEFAULT '0', + scorpia int DEFAULT '0', + skotizo int DEFAULT '0', + tempoross int DEFAULT '0', + the_gauntlet int DEFAULT '0', + the_corrupted_gauntlet int DEFAULT '0', + theatre_of_blood int DEFAULT '0', + theatre_of_blood_hard int DEFAULT '0', + thermonuclear_smoke_devil int DEFAULT '0', + tombs_of_amascut int DEFAULT '0', + tombs_of_amascut_expert int DEFAULT '0', + tzkal_zuk int DEFAULT '0', + tztok_jad int DEFAULT '0', + venenatis int DEFAULT '0', + vetion int DEFAULT '0', + vorkath int DEFAULT '0', + wintertodt int DEFAULT '0', + zalcano int DEFAULT '0', + zulrah int DEFAULT '0', + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + UNIQUE KEY idx_playerHiscoreData_Player_id_timestamp (Player_id,timestamp), + UNIQUE KEY Unique_player_date (Player_id,ts_date), + CONSTRAINT FK_Players_id FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +CREATE TABLE playerHiscoreDataLatest ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT NULL, + attack int DEFAULT NULL, + defence int DEFAULT NULL, + strength int DEFAULT NULL, + hitpoints int DEFAULT NULL, + ranged int DEFAULT NULL, + prayer int DEFAULT NULL, + magic int DEFAULT NULL, + cooking int DEFAULT NULL, + woodcutting int DEFAULT NULL, + fletching int DEFAULT NULL, + fishing int DEFAULT NULL, + firemaking int DEFAULT NULL, + crafting int DEFAULT NULL, + smithing int DEFAULT NULL, + mining int DEFAULT NULL, + herblore int DEFAULT NULL, + agility int DEFAULT NULL, + thieving int DEFAULT NULL, + slayer int DEFAULT NULL, + farming int DEFAULT NULL, + runecraft int DEFAULT NULL, + hunter int DEFAULT NULL, + construction int DEFAULT NULL, + league int DEFAULT NULL, + bounty_hunter_hunter int DEFAULT NULL, + bounty_hunter_rogue int DEFAULT NULL, + cs_all int DEFAULT NULL, + cs_beginner int DEFAULT NULL, + cs_easy int DEFAULT NULL, + cs_medium int DEFAULT NULL, + cs_hard int DEFAULT NULL, + cs_elite int DEFAULT NULL, + cs_master int DEFAULT NULL, + lms_rank int DEFAULT NULL, + soul_wars_zeal int DEFAULT NULL, + abyssal_sire int DEFAULT NULL, + alchemical_hydra int DEFAULT NULL, + barrows_chests int DEFAULT NULL, + bryophyta int DEFAULT NULL, + callisto int DEFAULT NULL, + cerberus int DEFAULT NULL, + chambers_of_xeric int DEFAULT NULL, + chambers_of_xeric_challenge_mode int DEFAULT NULL, + chaos_elemental int DEFAULT NULL, + chaos_fanatic int DEFAULT NULL, + commander_zilyana int DEFAULT NULL, + corporeal_beast int DEFAULT NULL, + crazy_archaeologist int DEFAULT NULL, + dagannoth_prime int DEFAULT NULL, + dagannoth_rex int DEFAULT NULL, + dagannoth_supreme int DEFAULT NULL, + deranged_archaeologist int DEFAULT NULL, + general_graardor int DEFAULT NULL, + giant_mole int DEFAULT NULL, + grotesque_guardians int DEFAULT NULL, + hespori int DEFAULT NULL, + kalphite_queen int DEFAULT NULL, + king_black_dragon int DEFAULT NULL, + kraken int DEFAULT NULL, + kreearra int DEFAULT NULL, + kril_tsutsaroth int DEFAULT NULL, + mimic int DEFAULT NULL, + nex int DEFAULT NULL, + nightmare int DEFAULT NULL, + phosanis_nightmare int DEFAULT NULL, + obor int DEFAULT NULL, + phantom_muspah int DEFAULT NULL, + sarachnis int DEFAULT NULL, + scorpia int DEFAULT NULL, + skotizo int DEFAULT NULL, + Tempoross int DEFAULT NULL, + the_gauntlet int DEFAULT NULL, + the_corrupted_gauntlet int DEFAULT NULL, + theatre_of_blood int DEFAULT NULL, + theatre_of_blood_hard int DEFAULT NULL, + thermonuclear_smoke_devil int DEFAULT NULL, + tombs_of_amascut int DEFAULT NULL, + tombs_of_amascut_expert int DEFAULT NULL, + tzkal_zuk int DEFAULT NULL, + tztok_jad int DEFAULT NULL, + venenatis int DEFAULT NULL, + vetion int DEFAULT NULL, + vorkath int DEFAULT NULL, + wintertodt int DEFAULT NULL, + zalcano int DEFAULT NULL, + zulrah int DEFAULT NULL, + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + UNIQUE KEY Unique_player (Player_id) USING BTREE, + UNIQUE KEY idx_playerHiscoreDataLatest_Player_id_timestamp (Player_id,timestamp), + UNIQUE KEY idx_playerHiscoreDataLatest_Player_id_ts_date (Player_id,ts_date), + CONSTRAINT FK_latest_player FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); +CREATE TABLE playerHiscoreDataXPChange ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT NULL, + attack int DEFAULT NULL, + defence int DEFAULT NULL, + strength int DEFAULT NULL, + hitpoints int DEFAULT NULL, + ranged int DEFAULT NULL, + prayer int DEFAULT NULL, + magic int DEFAULT NULL, + cooking int DEFAULT NULL, + woodcutting int DEFAULT NULL, + fletching int DEFAULT NULL, + fishing int DEFAULT NULL, + firemaking int DEFAULT NULL, + crafting int DEFAULT NULL, + smithing int DEFAULT NULL, + mining int DEFAULT NULL, + herblore int DEFAULT NULL, + agility int DEFAULT NULL, + thieving int DEFAULT NULL, + slayer int DEFAULT NULL, + farming int DEFAULT NULL, + runecraft int DEFAULT NULL, + hunter int DEFAULT NULL, + construction int DEFAULT NULL, + league int DEFAULT NULL, + bounty_hunter_hunter int DEFAULT NULL, + bounty_hunter_rogue int DEFAULT NULL, + cs_all int DEFAULT NULL, + cs_beginner int DEFAULT NULL, + cs_easy int DEFAULT NULL, + cs_medium int DEFAULT NULL, + cs_hard int DEFAULT NULL, + cs_elite int DEFAULT NULL, + cs_master int DEFAULT NULL, + lms_rank int DEFAULT NULL, + soul_wars_zeal int DEFAULT NULL, + abyssal_sire int DEFAULT NULL, + alchemical_hydra int DEFAULT NULL, + barrows_chests int DEFAULT NULL, + bryophyta int DEFAULT NULL, + callisto int DEFAULT NULL, + cerberus int DEFAULT NULL, + chambers_of_xeric int DEFAULT NULL, + chambers_of_xeric_challenge_mode int DEFAULT NULL, + chaos_elemental int DEFAULT NULL, + chaos_fanatic int DEFAULT NULL, + commander_zilyana int DEFAULT NULL, + corporeal_beast int DEFAULT NULL, + crazy_archaeologist int DEFAULT NULL, + dagannoth_prime int DEFAULT NULL, + dagannoth_rex int DEFAULT NULL, + dagannoth_supreme int DEFAULT NULL, + deranged_archaeologist int DEFAULT NULL, + general_graardor int DEFAULT NULL, + giant_mole int DEFAULT NULL, + grotesque_guardians int DEFAULT NULL, + hespori int DEFAULT NULL, + kalphite_queen int DEFAULT NULL, + king_black_dragon int DEFAULT NULL, + kraken int DEFAULT NULL, + kreearra int DEFAULT NULL, + kril_tsutsaroth int DEFAULT NULL, + mimic int DEFAULT NULL, + nex int DEFAULT NULL, + nightmare int DEFAULT NULL, + obor int DEFAULT NULL, + phantom_muspah int DEFAULT NULL, + phosanis_nightmare int DEFAULT NULL, + sarachnis int DEFAULT NULL, + scorpia int DEFAULT NULL, + skotizo int DEFAULT NULL, + Tempoross int DEFAULT NULL, + the_gauntlet int DEFAULT NULL, + the_corrupted_gauntlet int DEFAULT NULL, + theatre_of_blood int DEFAULT NULL, + theatre_of_blood_hard int DEFAULT NULL, + thermonuclear_smoke_devil int DEFAULT NULL, + tzkal_zuk int DEFAULT NULL, + tztok_jad int DEFAULT NULL, + venenatis int DEFAULT NULL, + vetion int DEFAULT NULL, + vorkath int DEFAULT NULL, + wintertodt int DEFAULT NULL, + zalcano int DEFAULT NULL, + zulrah int DEFAULT NULL, + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + KEY IDX_xpChange_Player_id_timestamp (Player_id,timestamp) USING BTREE, + KEY IDX_xpChange_Player_id_ts_date (Player_id,ts_date) USING BTREE, + CONSTRAINT fk_phd_xp_pl FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +CREATE TABLE `scraper_data` ( + `scraper_id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `player_id` int unsigned NOT NULL, + `record_date` date GENERATED ALWAYS AS (cast(`created_at` as date)) STORED, + PRIMARY KEY (`scraper_id`), + UNIQUE KEY `unique_player_per_day` (`player_id`,`record_date`) +); + +CREATE TABLE `scraper_data_latest` ( + `scraper_id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `record_date` date GENERATED ALWAYS AS (cast(`created_at` as date)) STORED, + `player_id` int unsigned NOT NULL, + PRIMARY KEY (`player_id`), + KEY `idx_scraper_id` (`scraper_id`), + KEY `idx_record_date` (`record_date`) +); + +CREATE TABLE skills ( + skill_id TINYINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, # < 255 + skill_name VARCHAR(50) NOT NULL, + UNIQUE KEY unique_skill_name (skill_name) +); +CREATE TABLE activities ( + activity_id TINYINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, # < 255 + activity_name VARCHAR(50) NOT NULL, + UNIQUE KEY unique_activity_name (activity_name) +); + + +CREATE TABLE player_skills ( + scraper_id BIGINT UNSIGNED NOT NULL, + skill_id TINYINT UNSIGNED NOT NULL, + skill_value INT UNSIGNED NOT NULL DEFAULT 0, # < 200 000 000 + FOREIGN KEY (scraper_id) REFERENCES scraper_data(scraper_id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(skill_id) ON DELETE CASCADE, + PRIMARY KEY (scraper_id, skill_id) +); + +CREATE TABLE player_activities ( + scraper_id BIGINT UNSIGNED NOT NULL, + activity_id TINYINT UNSIGNED NOT NULL, + activity_value INT UNSIGNED NOT NULL DEFAULT 0, # some guy could get over 65k kc + FOREIGN KEY (scraper_id) REFERENCES scraper_data(scraper_id) ON DELETE CASCADE, + FOREIGN KEY (activity_id) REFERENCES activities(activity_id) ON DELETE CASCADE, + PRIMARY KEY (scraper_id, activity_id) +); + +CREATE TABLE Labels ( + id INT NOT NULL AUTO_INCREMENT, + label VARCHAR(50) NOT NULL, + PRIMARY KEY (id), + UNIQUE INDEX Unique_label USING BTREE (label) VISIBLE +); + +DELIMITER // + +CREATE TRIGGER `sd_latest` AFTER INSERT ON `scraper_data` FOR EACH ROW +BEGIN + DECLARE latest_created_at DATETIME; + + -- Get the latest created_at from scraper_data_latest for the current player_id + SELECT created_at INTO latest_created_at + FROM scraper_data_latest + WHERE player_id = NEW.player_id; + + IF latest_created_at IS NULL THEN + INSERT INTO scraper_data_latest (scraper_id, created_at, player_id) + VALUES (NEW.scraper_id, NEW.created_at, NEW.player_id) + ON DUPLICATE KEY UPDATE + scraper_id = NEW.scraper_id, + created_at = NEW.created_at; + ELSEIF NEW.created_at > latest_created_at THEN + UPDATE scraper_data_latest + SET + scraper_id = NEW.scraper_id, + created_at = NEW.created_at + WHERE player_id = NEW.player_id; + END IF; +END // + +DELIMITER ; +-- ----------------------------------------------------- +-- Table `playerdata`.`apiPermissions` +-- ----------------------------------------------------- +CREATE TABLE `playerdata`.`apiPermissions` ( + `id` INT NOT NULL AUTO_INCREMENT, + `permission` TEXT NOT NULL, + PRIMARY KEY (`id`) +) +; + + +-- ----------------------------------------------------- +-- Table `playerdata`.`apiUser` +-- ----------------------------------------------------- +CREATE TABLE `playerdata`.`apiUser` ( + `id` INT NOT NULL AUTO_INCREMENT, + `username` TINYTEXT NOT NULL, + `token` TINYTEXT NOT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `last_used` DATETIME NULL DEFAULT NULL, + `ratelimit` INT NOT NULL DEFAULT '100', + `is_active` TINYINT(1) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + INDEX `idx_token` (`token`(15) ASC) VISIBLE +) +; + + +-- ----------------------------------------------------- +-- Table `playerdata`.`apiUsage` +-- ----------------------------------------------------- +CREATE TABLE `playerdata`.`apiUsage` ( + `id` BIGINT NOT NULL AUTO_INCREMENT, + `user_id` INT NOT NULL, + `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + `route` TEXT NULL DEFAULT NULL, + PRIMARY KEY (`id`), + INDEX `FK_apiUsage_apiUser` (`user_id` ASC) VISIBLE, + CONSTRAINT `FK_apiUsage_apiUser` + FOREIGN KEY (`user_id`) + REFERENCES `playerdata`.`apiUser` (`id`) + ON DELETE RESTRICT + ON UPDATE RESTRICT +); + + +-- ----------------------------------------------------- +-- Table `playerdata`.`apiUserPerms` +-- ----------------------------------------------------- +CREATE TABLE `playerdata`.`apiUserPerms` ( + `id` INT NOT NULL AUTO_INCREMENT, + `user_id` INT NOT NULL, + `permission_id` INT NOT NULL, + PRIMARY KEY (`id`), + INDEX `FK_apiUserPerms_apiUser` (`user_id` ASC) VISIBLE, + INDEX `FK_apiUserPerms_apiPermission` (`permission_id` ASC) VISIBLE, + CONSTRAINT `FK_apiUserPerms_apiPermission` + FOREIGN KEY (`permission_id`) + REFERENCES `playerdata`.`apiPermissions` (`id`) + ON DELETE RESTRICT + ON UPDATE RESTRICT, + CONSTRAINT `FK_apiUserPerms_apiUser` + FOREIGN KEY (`user_id`) + REFERENCES `playerdata`.`apiUser` (`id`) + ON DELETE RESTRICT + ON UPDATE RESTRICT +); diff --git a/mysql/docker-entrypoint-initdb.d/02_data.sql b/mysql/docker-entrypoint-initdb.d/02_data.sql new file mode 100644 index 0000000..282a77e --- /dev/null +++ b/mysql/docker-entrypoint-initdb.d/02_data.sql @@ -0,0 +1,167 @@ +USE playerdata; + +INSERT INTO playerdata.apiPermissions (permission) VALUES + ('request_highscores'), + ('verify_ban'), + ('create_token'), + ('verify_players'), + ('discord_general') +; +INSERT INTO playerdata.apiUser (username,token,last_used,ratelimit,is_active) VALUES + ('example','verify_ban',NULL,-1,1) +; +INSERT INTO playerdata.apiUserPerms (user_id,permission_id) VALUES + (1,1), + (1,2) +; + +INSERT INTO Labels (id, label) VALUES + (1, 'Real_Player'), + (4, 'Wintertodt_bot'), + (5, 'Mining_bot'), + (7, 'Hunter_bot'), + (8, 'Herblore_bot'), + (9, 'Fletching_bot'), + (10, 'Fishing_bot'), + (11, 'Crafting_bot'), + (12, 'Cooking_bot'), + (13, 'Woodcutting_bot'), + (15, 'Smithing_bot'), + (17, 'Magic_bot'), + (19, 'PVM_Ranged_Magic_bot'), + (21, 'Agility_bot'), + (27, 'Zalcano_bot'), + (38, 'Runecrafting_bot'), + (40, 'PVM_Ranged_bot'), + (41, 'PVM_Melee_bot'), + (42, 'Thieving_bot'), + (52, 'LMS_bot'), + (56, 'Fishing_Cooking_bot'), + (57, 'mort_myre_fungus_bot'), + (59, 'temp_real_player'), + (61, 'Soul_Wars_bot'), + (64, 'Construction_Magic_bot'), + (65, 'Vorkath_bot'), + (66, 'Clue_Scroll_bot'), + (67, 'Barrows_bot'), + (76, 'Woodcutting_Mining_bot'), + (77, 'Woodcutting_Firemaking_bot'), + (84, 'Mage_Guild_Store_bot'), + (87, 'Phosani_bot'), + (89, 'Unknown_bot'), + (90, 'Blast_mine_bot'), + (91, 'Zulrah_bot'), + (92, 'test_label'), + (109, 'Nex_bot'), + (110, 'Gauntlet_bot') +; + +INSERT INTO Labels(id, label) VALUES + (0, 'Unknown') +; +UPDATE `Labels` + set id=0 + where label='Unknown' +; + +-- Insert data into the Players table +INSERT INTO skills (skill_id, skill_name) VALUES + (2, 'attack'), + (3, 'defence'), + (4, 'strength'), + (5, 'hitpoints'), + (6, 'ranged'), + (7, 'prayer'), + (8, 'magic'), + (9, 'cooking'), + (10, 'woodcutting'), + (11, 'fletching'), + (12, 'fishing'), + (13, 'firemaking'), + (14, 'crafting'), + (15, 'smithing'), + (16, 'mining'), + (17, 'herblore'), + (18, 'agility'), + (19, 'thieving'), + (20, 'slayer'), + (21, 'farming'), + (22, 'runecraft'), + (23, 'hunter'), + (24, 'construction') +; + + +INSERT INTO activities (activity_id, activity_name) VALUES + (1, 'abyssal_sire'), + (2, 'alchemical_hydra'), + (3, 'artio'), + (4, 'barrows_chests'), + (5, 'bounty_hunter_hunter'), + (6, 'bounty_hunter_rogue'), + (7, 'bryophyta'), + (8, 'callisto'), + (9, 'calvarion'), + (10, 'cerberus'), + (11, 'chambers_of_xeric'), + (12, 'chambers_of_xeric_challenge_mode'), + (13, 'chaos_elemental'), + (14, 'chaos_fanatic'), + (15, 'commander_zilyana'), + (16, 'corporeal_beast'), + (17, 'crazy_archaeologist'), + (18, 'cs_all'), + (19, 'cs_beginner'), + (20, 'cs_easy'), + (21, 'cs_elite'), + (22, 'cs_hard'), + (23, 'cs_master'), + (24, 'cs_medium'), + (25, 'dagannoth_prime'), + (26, 'dagannoth_rex'), + (27, 'dagannoth_supreme'), + (28, 'deranged_archaeologist'), + (29, 'duke_sucellus'), + (30, 'general_graardor'), + (31, 'giant_mole'), + (32, 'grotesque_guardians'), + (33, 'hespori'), + (34, 'kalphite_queen'), + (35, 'king_black_dragon'), + (36, 'kraken'), + (37, 'kreearra'), + (38, 'kril_tsutsaroth'), + (39, 'league'), + (40, 'lms_rank'), + (41, 'mimic'), + (42, 'nex'), + (43, 'nightmare'), + (44, 'obor'), + (45, 'phantom_muspah'), + (46, 'phosanis_nightmare'), + (47, 'rifts_closed'), + (48, 'sarachnis'), + (49, 'scorpia'), + (50, 'skotizo'), + (51, 'soul_wars_zeal'), + (52, 'spindel'), + (53, 'tempoross'), + (54, 'the_corrupted_gauntlet'), + (55, 'the_gauntlet'), + (56, 'the_leviathan'), + (57, 'the_whisperer'), + (58, 'theatre_of_blood'), + (59, 'theatre_of_blood_hard'), + (60, 'thermonuclear_smoke_devil'), + (61, 'tombs_of_amascut'), + (62, 'tombs_of_amascut_expert'), + (63, 'tzkal_zuk'), + (64, 'tztok_jad'), + (65, 'vardorvis'), + (66, 'venenatis'), + (67, 'vetion'), + (68, 'vorkath'), + (69, 'wintertodt'), + (70, 'zalcano'), + (71, 'zulrah') +; \ No newline at end of file diff --git a/mysql_setup/Dockerfile b/mysql_setup/Dockerfile new file mode 100644 index 0000000..014cd75 --- /dev/null +++ b/mysql_setup/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "setup_mysql.py"] diff --git a/mysql_setup/requirements.txt b/mysql_setup/requirements.txt new file mode 100644 index 0000000..1ab8080 --- /dev/null +++ b/mysql_setup/requirements.txt @@ -0,0 +1,8 @@ +aiomysql==0.2.0 +cffi==1.16.0 +cryptography==42.0.5 +greenlet==3.0.3 +pycparser==2.22 +PyMySQL==1.1.0 +SQLAlchemy==2.0.28 +typing_extensions==4.10.0 diff --git a/mysql_setup/setup_mysql.py b/mysql_setup/setup_mysql.py new file mode 100644 index 0000000..f85ffa5 --- /dev/null +++ b/mysql_setup/setup_mysql.py @@ -0,0 +1,236 @@ +# script to insert all the data we need +import random +from datetime import datetime, timedelta + +from sqlalchemy import ( + Boolean, + Column, + Date, + DateTime, + ForeignKey, + Integer, + String, + create_engine, + func, +) +from sqlalchemy.dialects.mysql import BIGINT, SMALLINT, TINYINT +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +Base = declarative_base() +random.seed(42) + + +class Players(Base): + __tablename__ = "Players" + + id = Column(Integer, primary_key=True) + name = Column(String) + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow) + possible_ban = Column(Boolean, default=True) + confirmed_ban = Column(Boolean, default=False) + confirmed_player = Column(Boolean, default=False) + label_id = Column(Integer) + label_jagex = Column(Integer) + # ironman = Column(Boolean) + # hardcore_ironman = Column(Boolean) + # ultimate_ironman = Column(Boolean) + normalized_name = Column(String) + + +class ScraperData(Base): + __tablename__ = "scraper_data" + + scraper_id = Column(BIGINT, primary_key=True, autoincrement=True) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(SMALLINT, nullable=False) + record_date = Column(Date, nullable=True, server_default=func.current_date()) + + +class ScraperDataLatest(Base): + __tablename__ = "scraper_data_latest" + + scraper_id = Column(BIGINT) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(BIGINT, primary_key=True) + record_date = Column(Date, nullable=True, server_default=func.current_date()) + + +class Skills(Base): + __tablename__ = "skills" + + skill_id = Column(TINYINT, primary_key=True, autoincrement=True) + skill_name = Column(String(50), nullable=False) + + +class PlayerSkills(Base): + __tablename__ = "player_skills" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_id = Column( + TINYINT, + ForeignKey("skills.skill_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_value = Column(Integer, nullable=False, default=0) + + +class Activities(Base): + __tablename__ = "activities" + + activity_id = Column(TINYINT, primary_key=True, autoincrement=True) + activity_name = Column(String(50), nullable=False) + + +class PlayerActivities(Base): + __tablename__ = "player_activities" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_id = Column( + TINYINT, + ForeignKey("activities.activity_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_value = Column(Integer, nullable=False, default=0) + + +# Define other SQLAlchemy models for remaining tables in a similar manner + +# Create an engine and bind the base +engine = create_engine("mysql+pymysql://root:root_bot_buster@mysql:3306/playerdata") +Base.metadata.create_all(engine) + +# Create a session +Session = sessionmaker(bind=engine) +session = Session() + + +# Define function to generate random date within a year +def random_date(): + return datetime.utcnow() - timedelta(days=random.randint(0, 365)) + + +class Labels(Base): + __tablename__ = "Labels" + + id = Column(Integer, primary_key=True) + label = Column(String) + + +# Insert 'request_highscores' and 'verify_ban' into the labels table +labels_to_insert = ["request_highscores", "verify_ban"] +for label_name in labels_to_insert: + # Check if the label already exists + existing_label = session.query(Labels).filter_by(label=label_name).first() + if not existing_label: + label = Labels(label=label_name) + session.add(label) +session.commit() + +# Query the labels table to get all id values +label_ids = session.query(Labels.id).all() +label_ids = [id[0] for id in label_ids] # Convert list of tuples to list of ids + +# Insert data into Players table +len_players = 500 +for i in range(len_players): + print(f"Player_{i}") + # Check if the player already exists + existing_player = session.query(Players).filter_by(name=f"Player_{i}").first() + if not existing_player: + player = Players( + name=f"Player_{i}", + created_at=random_date(), + updated_at=random_date(), + possible_ban=random.choice([True, False]), + confirmed_ban=random.choice([True, False]), + confirmed_player=random.choice([True, False]), + label_id=random.choice(label_ids), # Select a random id from label_ids + label_jagex=random.randint(0, 2), + normalized_name=f"Player_{i}", + ) + session.add(player) +session.commit() + +# Insert data into Activities table before PlayerActivities +activity_names = [f"Activity_{i}" for i in range(1, 71)] +for activity_name in activity_names: + # Check if the activity already exists + existing_activity = ( + session.query(Activities).filter_by(activity_name=activity_name).first() + ) + if not existing_activity: + activity = Activities(activity_name=activity_name) + session.add(activity) +session.commit() + +skill_list = list(range(2, 24)) +activity_list = list(range(1, 71)) + +len_scraper_data = len_players * 3 + +# Insert data into Skills table before PlayerSkills +skill_names = [f"Skill_{i}" for i in range(1, 24)] +for skill_name in skill_names: + # Check if the skill already exists + existing_skill = session.query(Skills).filter_by(skill_name=skill_name).first() + if not existing_skill: + skill = Skills(skill_name=skill_name) + session.add(skill) +session.commit() + +# Query the skills table to get all id values +skill_ids = session.query(Skills.skill_id).all() +skill_ids = [id[0] for id in skill_ids] # Convert list of tuples to list of ids + +for i in range(1, len_scraper_data + 1): + print(f"scraper_data_{i}") + # pick random player + player_id = random.randint(1, len_players) + + # pick random amount of skills + amount_skills = random.randint(0, len(skill_ids)) + random.shuffle(skill_ids) + skills = skill_ids[:amount_skills] + + # pick random amount of activities + amount_activities = random.randint(0, len(activity_list)) + random.shuffle(activity_list) + activities = activity_list[:amount_activities] + + # scraper data + try: + session.add( + ScraperData(scraper_id=i, player_id=player_id, created_at=random_date()) + ) + session.commit() + for skill in skills: + session.add( + PlayerSkills( + scraper_id=i, + skill_id=skill, + skill_value=random.randint(1, 200_000_000), + ) + ) + for activity in activities: + session.add( + PlayerActivities( + scraper_id=i, + activity_id=activity, + activity_value=random.randint(1, 65_000), + ) + ) + except IntegrityError: + session.rollback() # Rollback the transaction if a duplicate entry is encountered + finally: + session.commit()