diff --git a/.vscode/settings.json b/.vscode/settings.json index d99f2f3..6b50668 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" - }, - "python.formatting.provider": "none" -} \ No newline at end of file + "editor.formatOnSave": true, + "editor.defaultFormatter": "charliermarsh.ruff" + } +} diff --git a/kafka_setup/setup_kafka.py b/kafka_setup/setup_kafka.py index 49beb82..54ce59d 100644 --- a/kafka_setup/setup_kafka.py +++ b/kafka_setup/setup_kafka.py @@ -4,7 +4,6 @@ from kafka import KafkaProducer import os import zipfile -import time def create_topics(): diff --git a/mysql/docker-entrypoint-initdb.d/01_tables.sql b/mysql/docker-entrypoint-initdb.d/01_tables.sql index a165872..16aa9c4 100644 --- a/mysql/docker-entrypoint-initdb.d/01_tables.sql +++ b/mysql/docker-entrypoint-initdb.d/01_tables.sql @@ -40,8 +40,8 @@ CREATE TABLE Reports ( equip_weapon_id INT, equip_shield_id INT, equip_ge_value BIGINT, - CONSTRAINT `FK_Reported_Players_id` FOREIGN KEY (`reportedID`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT, - CONSTRAINT `FK_Reporting_Players_id` FOREIGN KEY (`reportingID`) REFERENCES `Players` (`id`) ON DELETE RESTRICT ON UPDATE RESTRICT + CONSTRAINT FK_Reported_Players_id FOREIGN KEY (reportedID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT FK_Reporting_Players_id FOREIGN KEY (reportingID) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT ); @@ -77,3 +77,319 @@ CREATE TABLE Predictions ( herblore_bot DECIMAL(5, 2) DEFAULT 0, unknown_bot DECIMAL(5, 2) DEFAULT 0 ); + +CREATE TABLE playerHiscoreData ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT '0', + attack int DEFAULT '0', + defence int DEFAULT '0', + strength int DEFAULT '0', + hitpoints int DEFAULT '0', + ranged int DEFAULT '0', + prayer int DEFAULT '0', + magic int DEFAULT '0', + cooking int DEFAULT '0', + woodcutting int DEFAULT '0', + fletching int DEFAULT '0', + fishing int DEFAULT '0', + firemaking int DEFAULT '0', + crafting int DEFAULT '0', + smithing int DEFAULT '0', + mining int DEFAULT '0', + herblore int DEFAULT '0', + agility int DEFAULT '0', + thieving int DEFAULT '0', + slayer int DEFAULT '0', + farming int DEFAULT '0', + runecraft int DEFAULT '0', + hunter int DEFAULT '0', + construction int DEFAULT '0', + league int DEFAULT '0', + bounty_hunter_hunter int DEFAULT '0', + bounty_hunter_rogue int DEFAULT '0', + cs_all int DEFAULT '0', + cs_beginner int DEFAULT '0', + cs_easy int DEFAULT '0', + cs_medium int DEFAULT '0', + cs_hard int DEFAULT '0', + cs_elite int DEFAULT '0', + cs_master int DEFAULT '0', + lms_rank int DEFAULT '0', + soul_wars_zeal int DEFAULT '0', + abyssal_sire int DEFAULT '0', + alchemical_hydra int DEFAULT '0', + barrows_chests int DEFAULT '0', + bryophyta int DEFAULT '0', + callisto int DEFAULT '0', + cerberus int DEFAULT '0', + chambers_of_xeric int DEFAULT '0', + chambers_of_xeric_challenge_mode int DEFAULT '0', + chaos_elemental int DEFAULT '0', + chaos_fanatic int DEFAULT '0', + commander_zilyana int DEFAULT '0', + corporeal_beast int DEFAULT '0', + crazy_archaeologist int DEFAULT '0', + dagannoth_prime int DEFAULT '0', + dagannoth_rex int DEFAULT '0', + dagannoth_supreme int DEFAULT '0', + deranged_archaeologist int DEFAULT '0', + general_graardor int DEFAULT '0', + giant_mole int DEFAULT '0', + grotesque_guardians int DEFAULT '0', + hespori int DEFAULT '0', + kalphite_queen int DEFAULT '0', + king_black_dragon int DEFAULT '0', + kraken int DEFAULT '0', + kreearra int DEFAULT '0', + kril_tsutsaroth int DEFAULT '0', + mimic int DEFAULT '0', + nex int DEFAULT '0', + nightmare int DEFAULT '0', + phosanis_nightmare int DEFAULT '0', + obor int DEFAULT '0', + phantom_muspah int DEFAULT '0', + sarachnis int DEFAULT '0', + scorpia int DEFAULT '0', + skotizo int DEFAULT '0', + tempoross int DEFAULT '0', + the_gauntlet int DEFAULT '0', + the_corrupted_gauntlet int DEFAULT '0', + theatre_of_blood int DEFAULT '0', + theatre_of_blood_hard int DEFAULT '0', + thermonuclear_smoke_devil int DEFAULT '0', + tombs_of_amascut int DEFAULT '0', + tombs_of_amascut_expert int DEFAULT '0', + tzkal_zuk int DEFAULT '0', + tztok_jad int DEFAULT '0', + venenatis int DEFAULT '0', + vetion int DEFAULT '0', + vorkath int DEFAULT '0', + wintertodt int DEFAULT '0', + zalcano int DEFAULT '0', + zulrah int DEFAULT '0', + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + UNIQUE KEY idx_playerHiscoreData_Player_id_timestamp (Player_id,timestamp), + UNIQUE KEY Unique_player_date (Player_id,ts_date), + CONSTRAINT FK_Players_id FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); + +CREATE TABLE playerHiscoreDataLatest ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT NULL, + attack int DEFAULT NULL, + defence int DEFAULT NULL, + strength int DEFAULT NULL, + hitpoints int DEFAULT NULL, + ranged int DEFAULT NULL, + prayer int DEFAULT NULL, + magic int DEFAULT NULL, + cooking int DEFAULT NULL, + woodcutting int DEFAULT NULL, + fletching int DEFAULT NULL, + fishing int DEFAULT NULL, + firemaking int DEFAULT NULL, + crafting int DEFAULT NULL, + smithing int DEFAULT NULL, + mining int DEFAULT NULL, + herblore int DEFAULT NULL, + agility int DEFAULT NULL, + thieving int DEFAULT NULL, + slayer int DEFAULT NULL, + farming int DEFAULT NULL, + runecraft int DEFAULT NULL, + hunter int DEFAULT NULL, + construction int DEFAULT NULL, + league int DEFAULT NULL, + bounty_hunter_hunter int DEFAULT NULL, + bounty_hunter_rogue int DEFAULT NULL, + cs_all int DEFAULT NULL, + cs_beginner int DEFAULT NULL, + cs_easy int DEFAULT NULL, + cs_medium int DEFAULT NULL, + cs_hard int DEFAULT NULL, + cs_elite int DEFAULT NULL, + cs_master int DEFAULT NULL, + lms_rank int DEFAULT NULL, + soul_wars_zeal int DEFAULT NULL, + abyssal_sire int DEFAULT NULL, + alchemical_hydra int DEFAULT NULL, + barrows_chests int DEFAULT NULL, + bryophyta int DEFAULT NULL, + callisto int DEFAULT NULL, + cerberus int DEFAULT NULL, + chambers_of_xeric int DEFAULT NULL, + chambers_of_xeric_challenge_mode int DEFAULT NULL, + chaos_elemental int DEFAULT NULL, + chaos_fanatic int DEFAULT NULL, + commander_zilyana int DEFAULT NULL, + corporeal_beast int DEFAULT NULL, + crazy_archaeologist int DEFAULT NULL, + dagannoth_prime int DEFAULT NULL, + dagannoth_rex int DEFAULT NULL, + dagannoth_supreme int DEFAULT NULL, + deranged_archaeologist int DEFAULT NULL, + general_graardor int DEFAULT NULL, + giant_mole int DEFAULT NULL, + grotesque_guardians int DEFAULT NULL, + hespori int DEFAULT NULL, + kalphite_queen int DEFAULT NULL, + king_black_dragon int DEFAULT NULL, + kraken int DEFAULT NULL, + kreearra int DEFAULT NULL, + kril_tsutsaroth int DEFAULT NULL, + mimic int DEFAULT NULL, + nex int DEFAULT NULL, + nightmare int DEFAULT NULL, + phosanis_nightmare int DEFAULT NULL, + obor int DEFAULT NULL, + phantom_muspah int DEFAULT NULL, + sarachnis int DEFAULT NULL, + scorpia int DEFAULT NULL, + skotizo int DEFAULT NULL, + Tempoross int NOT NULL, + the_gauntlet int DEFAULT NULL, + the_corrupted_gauntlet int DEFAULT NULL, + theatre_of_blood int DEFAULT NULL, + theatre_of_blood_hard int DEFAULT NULL, + thermonuclear_smoke_devil int DEFAULT NULL, + tombs_of_amascut int DEFAULT NULL, + tombs_of_amascut_expert int DEFAULT NULL, + tzkal_zuk int DEFAULT NULL, + tztok_jad int DEFAULT NULL, + venenatis int DEFAULT NULL, + vetion int DEFAULT NULL, + vorkath int DEFAULT NULL, + wintertodt int DEFAULT NULL, + zalcano int DEFAULT NULL, + zulrah int DEFAULT NULL, + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + UNIQUE KEY Unique_player (Player_id) USING BTREE, + UNIQUE KEY idx_playerHiscoreDataLatest_Player_id_timestamp (Player_id,timestamp), + UNIQUE KEY idx_playerHiscoreDataLatest_Player_id_ts_date (Player_id,ts_date), + CONSTRAINT FK_latest_player FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); +CREATE TABLE playerHiscoreDataXPChange ( + id bigint NOT NULL AUTO_INCREMENT, + timestamp datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + ts_date date DEFAULT NULL, + Player_id int NOT NULL, + total bigint DEFAULT NULL, + attack int DEFAULT NULL, + defence int DEFAULT NULL, + strength int DEFAULT NULL, + hitpoints int DEFAULT NULL, + ranged int DEFAULT NULL, + prayer int DEFAULT NULL, + magic int DEFAULT NULL, + cooking int DEFAULT NULL, + woodcutting int DEFAULT NULL, + fletching int DEFAULT NULL, + fishing int DEFAULT NULL, + firemaking int DEFAULT NULL, + crafting int DEFAULT NULL, + smithing int DEFAULT NULL, + mining int DEFAULT NULL, + herblore int DEFAULT NULL, + agility int DEFAULT NULL, + thieving int DEFAULT NULL, + slayer int DEFAULT NULL, + farming int DEFAULT NULL, + runecraft int DEFAULT NULL, + hunter int DEFAULT NULL, + construction int DEFAULT NULL, + league int DEFAULT NULL, + bounty_hunter_hunter int DEFAULT NULL, + bounty_hunter_rogue int DEFAULT NULL, + cs_all int DEFAULT NULL, + cs_beginner int DEFAULT NULL, + cs_easy int DEFAULT NULL, + cs_medium int DEFAULT NULL, + cs_hard int DEFAULT NULL, + cs_elite int DEFAULT NULL, + cs_master int DEFAULT NULL, + lms_rank int DEFAULT NULL, + soul_wars_zeal int DEFAULT NULL, + abyssal_sire int DEFAULT NULL, + alchemical_hydra int DEFAULT NULL, + barrows_chests int DEFAULT NULL, + bryophyta int DEFAULT NULL, + callisto int DEFAULT NULL, + cerberus int DEFAULT NULL, + chambers_of_xeric int DEFAULT NULL, + chambers_of_xeric_challenge_mode int DEFAULT NULL, + chaos_elemental int DEFAULT NULL, + chaos_fanatic int DEFAULT NULL, + commander_zilyana int DEFAULT NULL, + corporeal_beast int DEFAULT NULL, + crazy_archaeologist int DEFAULT NULL, + dagannoth_prime int DEFAULT NULL, + dagannoth_rex int DEFAULT NULL, + dagannoth_supreme int DEFAULT NULL, + deranged_archaeologist int DEFAULT NULL, + general_graardor int DEFAULT NULL, + giant_mole int DEFAULT NULL, + grotesque_guardians int DEFAULT NULL, + hespori int DEFAULT NULL, + kalphite_queen int DEFAULT NULL, + king_black_dragon int DEFAULT NULL, + kraken int DEFAULT NULL, + kreearra int DEFAULT NULL, + kril_tsutsaroth int DEFAULT NULL, + mimic int DEFAULT NULL, + nex int DEFAULT NULL, + nightmare int DEFAULT NULL, + obor int DEFAULT NULL, + phantom_muspah int DEFAULT NULL, + phosanis_nightmare int DEFAULT NULL, + sarachnis int DEFAULT NULL, + scorpia int DEFAULT NULL, + skotizo int DEFAULT NULL, + Tempoross int DEFAULT NULL, + the_gauntlet int DEFAULT NULL, + the_corrupted_gauntlet int DEFAULT NULL, + theatre_of_blood int DEFAULT NULL, + theatre_of_blood_hard int DEFAULT NULL, + thermonuclear_smoke_devil int DEFAULT NULL, + tzkal_zuk int DEFAULT NULL, + tztok_jad int DEFAULT NULL, + venenatis int DEFAULT NULL, + vetion int DEFAULT NULL, + vorkath int DEFAULT NULL, + wintertodt int DEFAULT NULL, + zalcano int DEFAULT NULL, + zulrah int DEFAULT NULL, + rifts_closed int DEFAULT '0', + artio int DEFAULT '0', + calvarion int DEFAULT '0', + duke_sucellus int DEFAULT '0', + spindel int DEFAULT '0', + the_leviathan int DEFAULT '0', + the_whisperer int DEFAULT '0', + vardorvis int DEFAULT '0', + PRIMARY KEY (id), + KEY IDX_xpChange_Player_id_timestamp (Player_id,timestamp) USING BTREE, + KEY IDX_xpChange_Player_id_ts_date (Player_id,ts_date) USING BTREE, + CONSTRAINT fk_phd_xp_pl FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT +); diff --git a/notes.md b/notes.md index 79ab742..599cade 100644 --- a/notes.md +++ b/notes.md @@ -36,4 +36,5 @@ sudo apt install python3.10-venv -y ```sh python3 -m venv .venv touch .venv\bin\activate -``` \ No newline at end of file +``` +```sh diff --git a/src/api/v2/__init__.py b/src/api/v2/__init__.py index b5ae808..b8424c4 100644 --- a/src/api/v2/__init__.py +++ b/src/api/v2/__init__.py @@ -1,5 +1,6 @@ from fastapi import APIRouter -from . import player +from . import player, highscore router = APIRouter() -router.include_router(player.router) \ No newline at end of file +router.include_router(player.router) +router.include_router(highscore.router) diff --git a/src/api/v2/highscore.py b/src/api/v2/highscore.py new file mode 100644 index 0000000..b308369 --- /dev/null +++ b/src/api/v2/highscore.py @@ -0,0 +1,36 @@ +from fastapi import APIRouter, Query +from src.app.repositories.highscore import HighscoreLatest + +router = APIRouter() + + +@router.get("/highscore/latest") +async def get_highscore_latest( + player_id: int, + many: bool = False, + limit: int = Query(default=10, ge=0, le=10_000), +): + repo = HighscoreLatest() + if many: + data = await repo.get_many(start=player_id, limit=limit) + else: + data = await repo.get(id=player_id) + return data + + +# @router.get("/highscore") +# async def get_highscore( +# player_id: str = None, +# greater_than: bool = None, +# limit: int = Query(default=1_000, ge=0, le=10_000), +# ): +# return {} + + +# @router.get("/highscore/xp") +# async def get_highscore_xp( +# player_id: str = None, +# greater_than: bool = None, +# limit: int = Query(default=1_000, ge=0, le=10_000), +# ): +# return {} diff --git a/src/api/v2/player.py b/src/api/v2/player.py index bd49d85..36cb34a 100644 --- a/src/api/v2/player.py +++ b/src/api/v2/player.py @@ -1,5 +1,5 @@ from fastapi import APIRouter, Query -from src.app.models.player import Player +from src.app.repositories.player import Player router = APIRouter() @@ -11,8 +11,10 @@ async def get_player( greater_than: bool = None, limit: int = Query(default=1_000, ge=0, le=100_000), ): - player_model = Player() - data = await player_model.get_player( + # TODO: make use of abstract base class + repo = Player() + + data = await repo.get_player( player_id=player_id, player_name=player_name, greater_than=greater_than, diff --git a/src/app/models/.gitkeep b/src/app/repositories/.gitkeep similarity index 100% rename from src/app/models/.gitkeep rename to src/app/repositories/.gitkeep diff --git a/src/app/models/__init__.py b/src/app/repositories/__init__.py similarity index 100% rename from src/app/models/__init__.py rename to src/app/repositories/__init__.py diff --git a/src/app/repositories/abstract_repo.py b/src/app/repositories/abstract_repo.py new file mode 100644 index 0000000..fe94af2 --- /dev/null +++ b/src/app/repositories/abstract_repo.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod + +class AbstractAPI(ABC): + @abstractmethod + def get(self, id): + raise NotImplementedError + + @abstractmethod + def get_many(self, start, limit): + raise NotImplementedError + + @abstractmethod + def update(self, id, data): + raise NotImplementedError + + @abstractmethod + def delete(self, id): + raise NotImplementedError diff --git a/src/app/repositories/highscore.py b/src/app/repositories/highscore.py new file mode 100644 index 0000000..00f661f --- /dev/null +++ b/src/app/repositories/highscore.py @@ -0,0 +1,43 @@ +from src.core.database.models.highscore import ( + # playerHiscoreData, + PlayerHiscoreDataLatest, + # PlayerHiscoreDataXPChange, +) +from src.core.database.database import SessionFactory +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.sql.expression import Select +from fastapi.encoders import jsonable_encoder +from src.app.repositories.abstract_repo import AbstractAPI + + +class HighscoreLatest(AbstractAPI): + def __init__(self) -> None: + super().__init__() + self.table = PlayerHiscoreDataLatest + + async def _simple_execute(self, sql) -> dict: + async with SessionFactory() as session: + session: AsyncSession + + result: AsyncResult = await session.execute(sql) + result = result.scalars().all() + return jsonable_encoder(result) + + async def get(self, id: int): + sql: Select = select(self.table) + sql = sql.where(self.table.Player_id == id) + return await self._simple_execute(sql) + + async def get_many(self, start: int, limit: int = 5000): + sql: Select = select(self.table) + sql = sql.where(self.table.Player_id > start) + sql = sql.limit(limit) + return await self._simple_execute(sql) + + async def delete(self, id): + pass + + async def update(self, id, data): + pass diff --git a/src/app/models/player.py b/src/app/repositories/player.py similarity index 86% rename from src/app/models/player.py rename to src/app/repositories/player.py index 13d7d9b..ab17090 100644 --- a/src/app/models/player.py +++ b/src/app/repositories/player.py @@ -1,34 +1,34 @@ -from src.core.database.models.player import Player as dbPlayer -from src.core.database.database import SessionFactory -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import delete, insert, select, update -from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.sql.expression import Delete, Insert, Select, Update, and_ -from fastapi.encoders import jsonable_encoder - - -class Player: - def __init__(self) -> None: - pass - - async def get_player( - self, player_id: int, player_name: str, greater_than: bool, limit: int = 1_000 - ): - table = dbPlayer - sql_select: Select = select(table) - sql_select = sql_select.limit(limit) - - if player_name: - sql_select = sql_select.where(dbPlayer.name >= player_name) - - if greater_than: - sql_select = sql_select.where(dbPlayer.id >= player_id) - elif player_id: - sql_select = sql_select.where(dbPlayer.id == player_id) - - async with SessionFactory() as session: - session: AsyncSession - - result: AsyncResult = await session.execute(sql_select) - result = result.scalars().all() - return jsonable_encoder(result) +from src.core.database.models.player import Player as dbPlayer +from src.core.database.database import SessionFactory +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.sql.expression import Select +from fastapi.encoders import jsonable_encoder + + +class Player: + def __init__(self) -> None: + pass + + async def get_player( + self, player_id: int, player_name: str, greater_than: bool, limit: int = 1_000 + ): + table = dbPlayer + sql_select: Select = select(table) + sql_select = sql_select.limit(limit) + + if player_name: + sql_select = sql_select.where(dbPlayer.name >= player_name) + + if greater_than: + sql_select = sql_select.where(dbPlayer.id >= player_id) + elif player_id: + sql_select = sql_select.where(dbPlayer.id == player_id) + + async with SessionFactory() as session: + session: AsyncSession + + result: AsyncResult = await session.execute(sql_select) + result = result.scalars().all() + return jsonable_encoder(result) diff --git a/src/app/views/response/highscore.py b/src/app/views/response/highscore.py new file mode 100644 index 0000000..b5ad6bc --- /dev/null +++ b/src/app/views/response/highscore.py @@ -0,0 +1,106 @@ +from pydantic import BaseModel, ConfigDict +from datetime import datetime, date +from typing import Optional + +class PlayerHiscoreData(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: Optional[int] = None + timestamp: datetime = datetime.utcnow() + ts_date: Optional[date] = None + Player_id: int + total: int + attack: int + defence: int + strength: int + hitpoints: int + ranged: int + prayer: int + magic: int + cooking: int + woodcutting: int + fletching: int + fishing: int + firemaking: int + crafting: int + smithing: int + mining: int + herblore: int + agility: int + thieving: int + slayer: int + farming: int + runecraft: int + hunter: int + construction: int + league: int + bounty_hunter_hunter: int + bounty_hunter_rogue: int + cs_all: int + cs_beginner: int + cs_easy: int + cs_medium: int + cs_hard: int + cs_elite: int + cs_master: int + lms_rank: int + soul_wars_zeal: int + abyssal_sire: int + alchemical_hydra: int + barrows_chests: int + bryophyta: int + callisto: int + cerberus: int + chambers_of_xeric: int + chambers_of_xeric_challenge_mode: int + chaos_elemental: int + chaos_fanatic: int + commander_zilyana: int + corporeal_beast: int + crazy_archaeologist: int + dagannoth_prime: int + dagannoth_rex: int + dagannoth_supreme: int + deranged_archaeologist: int + general_graardor: int + giant_mole: int + grotesque_guardians: int + hespori: int + kalphite_queen: int + king_black_dragon: int + kraken: int + kreearra: int + kril_tsutsaroth: int + mimic: int + nightmare: int + nex: int = 0 + phosanis_nightmare: int + obor: int + phantom_muspah: int = 0 + sarachnis: int + scorpia: int + skotizo: int + tempoross: int = 0 + the_gauntlet: int + the_corrupted_gauntlet: int + theatre_of_blood: int + theatre_of_blood_hard: int = 0 + thermonuclear_smoke_devil: int + tombs_of_amascut: int = 0 + tombs_of_amascut_expert: int = 0 + tzkal_zuk: int + tztok_jad: int + venenatis: int + vetion: int + vorkath: int + wintertodt: int + zalcano: int + zulrah: int + rifts_closed: int = 0 + artio: int = 0 + calvarion: int = 0 + duke_sucellus: int = 0 + spindel: int = 0 + the_leviathan: int = 0 + the_whisperer: int = 0 + vardorvis: int = 0 diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..384d4b0 --- /dev/null +++ b/src/core/__init__.py @@ -0,0 +1 @@ +# needed for log formatting \ No newline at end of file diff --git a/src/core/database/database.py b/src/core/database/database.py index 07b7d54..dd7c750 100644 --- a/src/core/database/database.py +++ b/src/core/database/database.py @@ -1,4 +1,3 @@ -import sqlalchemy from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker diff --git a/src/core/database/models/highscore.py b/src/core/database/models/highscore.py new file mode 100644 index 0000000..b5e33f4 --- /dev/null +++ b/src/core/database/models/highscore.py @@ -0,0 +1,351 @@ +from sqlalchemy import ( + BigInteger, + Column, + Date, + DateTime, + ForeignKey, + Index, + Integer, + text, +) + +from sqlalchemy.ext.declarative import declarative_base + + +Base = declarative_base() +metadata = Base.metadata + +class playerHiscoreData(Base): + __tablename__ = "playerHiscoreData" + __table_args__ = ( + Index("Unique_player_time", "Player_id", "timestamp", unique=True), + Index("Unique_player_date", "Player_id", "ts_date", unique=True), + ) + + id = Column(Integer, primary_key=True, autoincrement=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + phosanis_nightmare = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + tempoross = Column(Integer) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) + + +class PlayerHiscoreDataLatest(Base): + __tablename__ = "playerHiscoreDataLatest" + + id = Column(Integer, primary_key=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + unique=True, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + phosanis_nightmare = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + Tempoross = Column(Integer, nullable=False) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) +class PlayerHiscoreDataXPChange(Base): + __tablename__ = "playerHiscoreDataXPChange" + + id = Column(Integer, primary_key=True) + timestamp = Column( + DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP") + ) + ts_date = Column(Date) + Player_id = Column( + ForeignKey("Players.id", ondelete="RESTRICT", onupdate="RESTRICT"), + nullable=False, + index=True, + ) + total = Column(BigInteger) + attack = Column(Integer) + defence = Column(Integer) + strength = Column(Integer) + hitpoints = Column(Integer) + ranged = Column(Integer) + prayer = Column(Integer) + magic = Column(Integer) + cooking = Column(Integer) + woodcutting = Column(Integer) + fletching = Column(Integer) + fishing = Column(Integer) + firemaking = Column(Integer) + crafting = Column(Integer) + smithing = Column(Integer) + mining = Column(Integer) + herblore = Column(Integer) + agility = Column(Integer) + thieving = Column(Integer) + slayer = Column(Integer) + farming = Column(Integer) + runecraft = Column(Integer) + hunter = Column(Integer) + construction = Column(Integer) + league = Column(Integer) + bounty_hunter_hunter = Column(Integer) + bounty_hunter_rogue = Column(Integer) + cs_all = Column(Integer) + cs_beginner = Column(Integer) + cs_easy = Column(Integer) + cs_medium = Column(Integer) + cs_hard = Column(Integer) + cs_elite = Column(Integer) + cs_master = Column(Integer) + lms_rank = Column(Integer) + soul_wars_zeal = Column(Integer) + abyssal_sire = Column(Integer) + alchemical_hydra = Column(Integer) + barrows_chests = Column(Integer) + bryophyta = Column(Integer) + callisto = Column(Integer) + cerberus = Column(Integer) + chambers_of_xeric = Column(Integer) + chambers_of_xeric_challenge_mode = Column(Integer) + chaos_elemental = Column(Integer) + chaos_fanatic = Column(Integer) + commander_zilyana = Column(Integer) + corporeal_beast = Column(Integer) + crazy_archaeologist = Column(Integer) + dagannoth_prime = Column(Integer) + dagannoth_rex = Column(Integer) + dagannoth_supreme = Column(Integer) + deranged_archaeologist = Column(Integer) + general_graardor = Column(Integer) + giant_mole = Column(Integer) + grotesque_guardians = Column(Integer) + hespori = Column(Integer) + kalphite_queen = Column(Integer) + king_black_dragon = Column(Integer) + kraken = Column(Integer) + kreearra = Column(Integer) + kril_tsutsaroth = Column(Integer) + mimic = Column(Integer) + nightmare = Column(Integer) + nex = Column(Integer) + obor = Column(Integer) + phantom_muspah = Column(Integer) + phosanis_nightmare = Column(Integer) + sarachnis = Column(Integer) + scorpia = Column(Integer) + skotizo = Column(Integer) + Tempoross = Column(Integer, nullable=False) + the_gauntlet = Column(Integer) + the_corrupted_gauntlet = Column(Integer) + theatre_of_blood = Column(Integer) + theatre_of_blood_hard = Column(Integer) + thermonuclear_smoke_devil = Column(Integer) + tombs_of_amascut = Column(Integer) + tombs_of_amascut_expert = Column(Integer) + tzkal_zuk = Column(Integer) + tztok_jad = Column(Integer) + venenatis = Column(Integer) + vetion = Column(Integer) + vorkath = Column(Integer) + wintertodt = Column(Integer) + zalcano = Column(Integer) + zulrah = Column(Integer) + # New columns added + rifts_closed = Column(Integer, default=0) + artio = Column(Integer, default=0) + calvarion = Column(Integer, default=0) + duke_sucellus = Column(Integer, default=0) + spindel = Column(Integer, default=0) + the_leviathan = Column(Integer, default=0) + the_whisperer = Column(Integer, default=0) + vardorvis = Column(Integer, default=0) \ No newline at end of file diff --git a/src/core/logging_config.py b/src/core/logging.py similarity index 92% rename from src/core/logging_config.py rename to src/core/logging.py index d9918b7..46f8f95 100644 --- a/src/core/logging_config.py +++ b/src/core/logging.py @@ -1,43 +1,41 @@ -import json -import logging -import sys -import warnings - -from .config import settings - -# # log formatting -formatter = logging.Formatter( - json.dumps( - { - "ts": "%(asctime)s", - "name": "%(name)s", - "function": "%(funcName)s", - "level": "%(levelname)s", - "msg": json.dumps("%(message)s"), - } - ) -) - -stream_handler = logging.StreamHandler(sys.stdout) - -stream_handler.setFormatter(formatter) - -handlers = [stream_handler] - -logging.basicConfig(level=logging.DEBUG, handlers=handlers) - -# set imported loggers to warning -logging.getLogger("urllib3").setLevel(logging.DEBUG) -logging.getLogger("uvicorn").setLevel(logging.DEBUG) -logging.getLogger("aiomysql").setLevel(logging.ERROR) -logging.getLogger("aiokafka").setLevel(logging.WARNING) - -# if settings.ENV == "PRD": -# uvicorn_error = logging.getLogger("uvicorn.error") -# uvicorn_error.disabled = True -# uvicorn_access = logging.getLogger("uvicorn.access") -# uvicorn_access.disabled = True - -# # https://github.com/aio-libs/aiomysql/issues/103 -# # https://github.com/coleifer/peewee/issues/2229 -# warnings.filterwarnings("ignore", ".*Duplicate entry.*") +import json +import logging +import sys + + +# # log formatting +formatter = logging.Formatter( + json.dumps( + { + "ts": "%(asctime)s", + "name": "%(name)s", + "function": "%(funcName)s", + "level": "%(levelname)s", + "msg": json.dumps("%(message)s"), + } + ) +) + +stream_handler = logging.StreamHandler(sys.stdout) + +stream_handler.setFormatter(formatter) + +handlers = [stream_handler] + +logging.basicConfig(level=logging.DEBUG, handlers=handlers) + +# set imported loggers to warning +logging.getLogger("urllib3").setLevel(logging.DEBUG) +logging.getLogger("uvicorn").setLevel(logging.DEBUG) +logging.getLogger("aiomysql").setLevel(logging.ERROR) +logging.getLogger("aiokafka").setLevel(logging.WARNING) + +# if settings.ENV == "PRD": +# uvicorn_error = logging.getLogger("uvicorn.error") +# uvicorn_error.disabled = True +# uvicorn_access = logging.getLogger("uvicorn.access") +# uvicorn_access.disabled = True + +# # https://github.com/aio-libs/aiomysql/issues/103 +# # https://github.com/coleifer/peewee/issues/2229 +# warnings.filterwarnings("ignore", ".*Duplicate entry.*") diff --git a/src/core/server.py b/src/core/server.py index ed83ec3..8abe404 100644 --- a/src/core/server.py +++ b/src/core/server.py @@ -1,4 +1,3 @@ -import asyncio import logging from fastapi import FastAPI @@ -8,7 +7,7 @@ from src import api from src.core.fastapi.middleware.logging import LoggingMiddleware -from . import logging_config # needed for log formatting + logger = logging.getLogger(__name__)