From 4d70b82d6d37bdff3a0d9e9bbd8efdc22483363b Mon Sep 17 00:00:00 2001 From: extreme4all <40169115+extreme4all@users.noreply.github.com> Date: Sun, 24 Mar 2024 22:27:26 +0100 Subject: [PATCH] support new data model (#14) * Bump cryptography from 41.0.4 to 42.0.4 (#13) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.4 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.4...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump fastapi from 0.103.1 to 0.109.1 (#15) Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.103.1 to 0.109.1. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.103.1...0.109.1) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump starlette from 0.27.0 to 0.36.2 (#16) Bumps [starlette](https://github.com/encode/starlette) from 0.27.0 to 0.36.2. - [Release notes](https://github.com/encode/starlette/releases) - [Changelog](https://github.com/encode/starlette/blob/master/docs/release-notes.md) - [Commits](https://github.com/encode/starlette/compare/0.27.0...0.36.2) --- updated-dependencies: - dependency-name: starlette dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * add tables * testing & new data * added skills * added activity --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: extreme4all <> --- .pre-commit-config.yaml | 17 + Makefile | 27 +- docker-compose.yaml | 52 +- .../docker-entrypoint-initdb.d/01_tables.sql | 49 ++ mysql/docker-entrypoint-initdb.d/02_data.sql | 631 +++++++++++++++++- notes.md | 27 +- requirements.txt | 47 +- src/api/__init__.py | 4 +- src/api/v2/highscore.py | 34 +- src/api/v2/player.py | 13 +- src/api/v3/__init__.py | 6 + src/api/v3/highscore.py | 45 ++ src/app/repositories/__init__.py | 15 + src/app/repositories/abstract_repo.py | 9 +- src/app/repositories/highscore.py | 87 ++- src/app/repositories/player.py | 44 +- src/app/repositories/player_activities.py | 49 ++ src/app/repositories/player_skills.py | 49 ++ src/app/repositories/scraper_data.py | 79 +++ src/core/.gitkeep | 0 src/core/__init__.py | 5 +- src/core/database/database.py | 3 +- src/core/database/models/__init__.py | 26 + src/core/database/models/activities.py | 27 + src/core/database/models/highscore.py | 9 +- src/core/database/models/scraper_data.py | 22 + src/core/database/models/skills.py | 27 + src/core/{logging.py => logging_config.py} | 9 +- src/core/server.py | 14 +- tests/conftest.py | 58 ++ tests/pytest.ini | 2 + tests/test_highscore.py | 44 ++ tests/test_player.py | 76 +++ 33 files changed, 1376 insertions(+), 230 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 src/api/v3/__init__.py create mode 100644 src/api/v3/highscore.py create mode 100644 src/app/repositories/player_activities.py create mode 100644 src/app/repositories/player_skills.py create mode 100644 src/app/repositories/scraper_data.py delete mode 100644 src/core/.gitkeep create mode 100644 src/core/database/models/__init__.py create mode 100644 src/core/database/models/activities.py create mode 100644 src/core/database/models/scraper_data.py create mode 100644 src/core/database/models/skills.py rename src/core/{logging.py => logging_config.py} (80%) create mode 100644 tests/conftest.py create mode 100644 tests/pytest.ini create mode 100644 tests/test_highscore.py create mode 100644 tests/test_player.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a4e23cc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black"] diff --git a/Makefile b/Makefile index 1fc473b..787ce68 100644 --- a/Makefile +++ b/Makefile @@ -58,17 +58,26 @@ pre-commit: ## Run pre-commit test-setup: python3 -m pip install pytest +create-venv: + python3 -m venv .venv + source .venv/bin/activate + requirements: python3 -m pip install -r requirements.txt + python3 -m pip install pytest-asyncio==0.23.6 + python3 -m pip install httpx==0.27.0 + python3 -m pip install pre-commit==3.6.2 + python3 -m pip install ruff==0.1.15 + pre-commit install -docker-down: - docker-compose down - -docker-rebuild: docker-down - docker-compose --verbose up --build +docker-restart: + docker compose down + docker compose up --build -d -docker-force-rebuild: - docker-compose --verbose up --build --force-recreate +docker test: + docker compose down + docker compose up --build -d + pytest api-setup: python3 -m pip install "fastapi[all]" @@ -76,7 +85,7 @@ api-setup: env-setup: touch .env echo "KAFKA_HOST= 'localhost:9092'" >> .env - echo "DATABASE_URL= 'mysql+aiomysql://root:root_bot_buster@localhost:3306/playerdata'" >> .env + echo "DATABASE_URL= 'mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata'" >> .env echo "ENV='DEV'" >> .env echo "POOL_RECYCLE='60'" >> .env echo "POOL_TIMEOUT='30'" >> .env @@ -84,4 +93,4 @@ env-setup: docs: open http://localhost:5000/docs xdg-open http://localhost:5000/docs - . http://localhost:5000/docs \ No newline at end of file + . http://localhost:5000/docs diff --git a/docker-compose.yaml b/docker-compose.yaml index 43cd0e1..8b94b9d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,55 +1,5 @@ version: '3' services: - # kafka: - # container_name: kafka - # image: bitnami/kafka:3.5.1-debian-11-r3 - # environment: - # - ALLOW_PLAINTEXT_LISTENER=yes - # - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - # - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT - # - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 - # - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false - # # volumes: - # # - ./kafka:/bitnami/kafka:rw - # ports: - # - 9094:9094 - # - 9092:9092 - # healthcheck: - # test: ["CMD", "kafka-topics.sh", "--list", "--bootstrap-server", "localhost:9092"] - # interval: 30s - # timeout: 10s - # retries: 5 - # networks: - # - botdetector-network - - # kafdrop: - # container_name: kafdrop - # image: obsidiandynamics/kafdrop:latest - # environment: - # - KAFKA_BROKERCONNECT=kafka:9092 - # - JVM_OPTS=-Xms32M -Xmx64M - # - SERVER_SERVLET_CONTEXTPATH=/ - # ports: - # - 9000:9000 - # restart: on-failure - # networks: - # - botdetector-network - # depends_on: - # kafka: - # condition: service_healthy - - # kafka_setup: - # container_name: kafka_setup - # build: - # context: ./kafka_setup - # environment: - # - KAFKA_BROKER=kafka:9092 - # networks: - # - botdetector-network - # depends_on: - # kafka: - # condition: service_healthy - mysql: container_name: database build: @@ -61,7 +11,7 @@ services: - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d # - ./mysql/mount:/var/lib/mysql # creates persistence ports: - - 3306:3306 + - 3307:3306 networks: - botdetector-network diff --git a/mysql/docker-entrypoint-initdb.d/01_tables.sql b/mysql/docker-entrypoint-initdb.d/01_tables.sql index 4a36892..02ac097 100644 --- a/mysql/docker-entrypoint-initdb.d/01_tables.sql +++ b/mysql/docker-entrypoint-initdb.d/01_tables.sql @@ -393,3 +393,52 @@ CREATE TABLE playerHiscoreDataXPChange ( KEY IDX_xpChange_Player_id_ts_date (Player_id,ts_date) USING BTREE, CONSTRAINT fk_phd_xp_pl FOREIGN KEY (Player_id) REFERENCES Players (id) ON DELETE RESTRICT ON UPDATE RESTRICT ); + +CREATE TABLE `scraper_data` ( + `scraper_id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `player_id` int unsigned NOT NULL, + `record_date` date GENERATED ALWAYS AS (cast(`created_at` as date)) STORED, + PRIMARY KEY (`scraper_id`), + UNIQUE KEY `unique_player_per_day` (`player_id`,`record_date`) +); + +CREATE TABLE `scraper_data_latest` ( + `scraper_id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `record_date` date GENERATED ALWAYS AS (cast(`created_at` as date)) STORED, + `player_id` int unsigned NOT NULL, + PRIMARY KEY (`player_id`), + KEY `idx_scraper_id` (`scraper_id`), + KEY `idx_record_date` (`record_date`) +); + +CREATE TABLE skills ( + skill_id TINYINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, # < 255 + skill_name VARCHAR(50) NOT NULL, + UNIQUE KEY unique_skill_name (skill_name) +); +CREATE TABLE activities ( + activity_id TINYINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, # < 255 + activity_name VARCHAR(50) NOT NULL, + UNIQUE KEY unique_activity_name (activity_name) +); + + +CREATE TABLE player_skills ( + scraper_id BIGINT UNSIGNED NOT NULL, + skill_id TINYINT UNSIGNED NOT NULL, + skill_value INT UNSIGNED NOT NULL DEFAULT 0, # < 200 000 000 + FOREIGN KEY (scraper_id) REFERENCES scraper_data(scraper_id) ON DELETE CASCADE, + FOREIGN KEY (skill_id) REFERENCES skills(skill_id) ON DELETE CASCADE, + PRIMARY KEY (scraper_id, skill_id) +); + +CREATE TABLE player_activities ( + scraper_id BIGINT UNSIGNED NOT NULL, + activity_id TINYINT UNSIGNED NOT NULL, + activity_value INT UNSIGNED NOT NULL DEFAULT 0, # some guy could get over 65k kc + FOREIGN KEY (scraper_id) REFERENCES scraper_data(scraper_id) ON DELETE CASCADE, + FOREIGN KEY (activity_id) REFERENCES activities(activity_id) ON DELETE CASCADE, + PRIMARY KEY (scraper_id, activity_id) +); diff --git a/mysql/docker-entrypoint-initdb.d/02_data.sql b/mysql/docker-entrypoint-initdb.d/02_data.sql index 8109db1..30c898b 100644 --- a/mysql/docker-entrypoint-initdb.d/02_data.sql +++ b/mysql/docker-entrypoint-initdb.d/02_data.sql @@ -287,7 +287,7 @@ LIMIT DELIMITER $$ -CREATE PROCEDURE INSERTROWS(NUM INT) BEGIN +CREATE PROCEDURE INSERTROWS(NUM INT) BEGIN DECLARE i INT; SET i = 1; WHILE i <= num DO SET @rand = FLOOR(1 + RAND() * 25); @@ -351,7 +351,7 @@ CREATE PROCEDURE INSERTROWS(NUM INT) BEGIN unknown_bot = IF (@rand = 25, 1, 0) * @multiplier; SET i = i + 1; END WHILE; - END$$ + END$$ DELIMITER ; @@ -371,3 +371,630 @@ VALUES (8, NOW(), CURDATE(), 8, 1700, 95, 85, 105, 100, 90, 95), (9, NOW(), CURDATE(), 9, 1000, 70, 60, 80, 75, 65, 70), (10, NOW(), CURDATE(), 10, 1200, 80, 70, 90, 85, 75, 80); + + +INSERT INTO skills (skill_id, skill_name) VALUES + (2, 'attack'), + (3, 'defence'), + (4, 'strength'), + (5, 'hitpoints'), + (6, 'ranged'), + (7, 'prayer'), + (8, 'magic'), + (9, 'cooking'), + (10, 'woodcutting'), + (11, 'fletching'), + (12, 'fishing'), + (13, 'firemaking'), + (14, 'crafting'), + (15, 'smithing'), + (16, 'mining'), + (17, 'herblore'), + (18, 'agility'), + (19, 'thieving'), + (20, 'slayer'), + (21, 'farming'), + (22, 'runecraft'), + (23, 'hunter'), + (24, 'construction') +; + + +INSERT INTO activities (activity_id, activity_name) VALUES + (1, 'abyssal_sire'), + (2, 'alchemical_hydra'), + (3, 'artio'), + (4, 'barrows_chests'), + (5, 'bounty_hunter_hunter'), + (6, 'bounty_hunter_rogue'), + (7, 'bryophyta'), + (8, 'callisto'), + (9, 'calvarion'), + (10, 'cerberus'), + (11, 'chambers_of_xeric'), + (12, 'chambers_of_xeric_challenge_mode'), + (13, 'chaos_elemental'), + (14, 'chaos_fanatic'), + (15, 'commander_zilyana'), + (16, 'corporeal_beast'), + (17, 'crazy_archaeologist'), + (18, 'cs_all'), + (19, 'cs_beginner'), + (20, 'cs_easy'), + (21, 'cs_elite'), + (22, 'cs_hard'), + (23, 'cs_master'), + (24, 'cs_medium'), + (25, 'dagannoth_prime'), + (26, 'dagannoth_rex'), + (27, 'dagannoth_supreme'), + (28, 'deranged_archaeologist'), + (29, 'duke_sucellus'), + (30, 'general_graardor'), + (31, 'giant_mole'), + (32, 'grotesque_guardians'), + (33, 'hespori'), + (34, 'kalphite_queen'), + (35, 'king_black_dragon'), + (36, 'kraken'), + (37, 'kreearra'), + (38, 'kril_tsutsaroth'), + (39, 'league'), + (40, 'lms_rank'), + (41, 'mimic'), + (42, 'nex'), + (43, 'nightmare'), + (44, 'obor'), + (45, 'phantom_muspah'), + (46, 'phosanis_nightmare'), + (47, 'rifts_closed'), + (48, 'sarachnis'), + (49, 'scorpia'), + (50, 'skotizo'), + (51, 'soul_wars_zeal'), + (52, 'spindel'), + (53, 'tempoross'), + (54, 'the_corrupted_gauntlet'), + (55, 'the_gauntlet'), + (56, 'the_leviathan'), + (57, 'the_whisperer'), + (58, 'theatre_of_blood'), + (59, 'theatre_of_blood_hard'), + (60, 'thermonuclear_smoke_devil'), + (61, 'tombs_of_amascut'), + (62, 'tombs_of_amascut_expert'), + (63, 'tzkal_zuk'), + (64, 'tztok_jad'), + (65, 'vardorvis'), + (66, 'venenatis'), + (67, 'vetion'), + (68, 'vorkath'), + (69, 'wintertodt'), + (70, 'zalcano'), + (71, 'zulrah') +; + +insert into scraper_data (scraper_id, created_at, player_id) VALUES + (4941843,'2024-03-18 02:13:46',1), + (7147825,'2024-03-19 00:49:45',1), + (9462081,'2024-03-20 00:48:29',1), + (12935727,'2024-03-21 00:11:55',1), + (14602384,'2024-03-22 04:40:49',1), + (15121534,'2024-03-23 00:10:05',1), + (1940168,'2024-03-17 00:05:17',8), + (4941785,'2024-03-18 02:13:46',8), + (7005058,'2024-03-19 00:05:31',8), + (9529321,'2024-03-20 00:48:31',8), + (13210211,'2024-03-21 00:46:29',8), + (14377161,'2024-03-22 00:28:25',8), + (15114040,'2024-03-23 00:10:04',8) +; +insert into scraper_data_latest (scraper_id, created_at, player_id) VALUES + (15121534,'2024-03-23 00:10:05',1), + (15114040,'2024-03-23 00:10:04',8) +; + +INSERT INTO player_activities (scraper_id, activity_id, activity_value) VALUES + (4941843,4,13), + (4941843,30,8), + (4941843,48,50), + (4941843,69,52), + (7147825,4,13), + (7147825,30,8), + (7147825,48,50), + (7147825,69,52), + (9462081,4,13), + (9462081,30,8), + (9462081,48,50), + (9462081,69,52), + (12935727,4,13), + (12935727,30,8), + (12935727,48,50), + (12935727,69,52), + (14602384,4,13), + (14602384,30,8), + (14602384,48,50), + (14602384,69,52), + (15121534,4,13), + (15121534,30,8), + (15121534,48,50), + (15121534,69,52), + (1940168,4,51), + (1940168,7,10), + (1940168,11,56), + (1940168,13,25), + (1940168,14,56), + (1940168,16,24), + (1940168,17,51), + (1940168,18,127), + (1940168,20,4), + (1940168,21,3), + (1940168,22,2), + (1940168,24,118), + (1940168,28,50), + (1940168,30,30), + (1940168,31,433), + (1940168,33,25), + (1940168,35,56), + (1940168,40,535), + (1940168,42,21), + (1940168,43,55), + (1940168,44,10), + (1940168,47,22), + (1940168,48,58), + (1940168,50,10), + (1940168,51,835), + (1940168,53,72), + (1940168,64,10), + (1940168,68,114), + (1940168,69,73), + (1940168,71,1256), + (4941785,4,51), + (4941785,7,10), + (4941785,11,56), + (4941785,13,25), + (4941785,14,56), + (4941785,16,24), + (4941785,17,51), + (4941785,18,127), + (4941785,20,4), + (4941785,21,3), + (4941785,22,2), + (4941785,24,118), + (4941785,28,50), + (4941785,30,30), + (4941785,31,433), + (4941785,33,25), + (4941785,35,56), + (4941785,40,535), + (4941785,42,21), + (4941785,43,55), + (4941785,44,10), + (4941785,47,22), + (4941785,48,58), + (4941785,50,10), + (4941785,51,835), + (4941785,53,72), + (4941785,64,10), + (4941785,68,114), + (4941785,69,73), + (4941785,71,1256), + (7005058,4,51), + (7005058,7,10), + (7005058,11,56), + (7005058,13,25), + (7005058,14,56), + (7005058,16,24), + (7005058,17,51), + (7005058,18,127), + (7005058,20,4), + (7005058,21,3), + (7005058,22,2), + (7005058,24,118), + (7005058,28,50), + (7005058,30,30), + (7005058,31,433), + (7005058,33,25), + (7005058,35,56), + (7005058,40,535), + (7005058,42,21), + (7005058,43,55), + (7005058,44,10), + (7005058,47,22), + (7005058,48,58), + (7005058,50,10), + (7005058,51,835), + (7005058,53,72), + (7005058,64,10), + (7005058,68,114), + (7005058,69,73), + (7005058,71,1256), + (9529321,4,51), + (9529321,7,10), + (9529321,11,56), + (9529321,13,25), + (9529321,14,56), + (9529321,16,24), + (9529321,17,51), + (9529321,18,127), + (9529321,20,4), + (9529321,21,3), + (9529321,22,2), + (9529321,24,118), + (9529321,28,50), + (9529321,30,30), + (9529321,31,433), + (9529321,33,25), + (9529321,35,56), + (9529321,40,535), + (9529321,42,21), + (9529321,43,55), + (9529321,44,10), + (9529321,47,22), + (9529321,48,58), + (9529321,50,10), + (9529321,51,835), + (9529321,53,72), + (9529321,64,10), + (9529321,68,114), + (9529321,69,73), + (9529321,71,1256), + (13210211,4,51), + (13210211,7,10), + (13210211,11,56), + (13210211,13,25), + (13210211,14,56), + (13210211,16,24), + (13210211,17,51), + (13210211,18,127), + (13210211,20,4), + (13210211,21,3), + (13210211,22,2), + (13210211,24,118), + (13210211,28,50), + (13210211,30,30), + (13210211,31,433), + (13210211,33,25), + (13210211,35,56), + (13210211,40,535), + (13210211,42,21), + (13210211,43,55), + (13210211,44,10), + (13210211,47,22), + (13210211,48,58), + (13210211,50,10), + (13210211,51,835), + (13210211,53,72), + (13210211,64,10), + (13210211,68,114), + (13210211,69,73), + (13210211,71,1256), + (14377161,4,51), + (14377161,7,10), + (14377161,11,56), + (14377161,13,25), + (14377161,14,56), + (14377161,16,24), + (14377161,17,51), + (14377161,18,127), + (14377161,20,4), + (14377161,21,3), + (14377161,22,2), + (14377161,24,118), + (14377161,28,50), + (14377161,30,30), + (14377161,31,433), + (14377161,33,25), + (14377161,35,56), + (14377161,40,535), + (14377161,42,21), + (14377161,43,55), + (14377161,44,10), + (14377161,47,22), + (14377161,48,58), + (14377161,50,10), + (14377161,51,835), + (14377161,53,72) +; + +INSERT INTO player_skills (scraper_id, skill_id, skill_value) VALUES + (4941843,2,2231), + (4941843,3,321), + (4941843,4,456), + (4941843,5,5), + (4941843,6,6), + (4941843,7,7), + (4941843,8,8), + (4941843,9,9), + (4941843,10,10), + (4941843,11,11), + (4941843,12,12), + (4941843,13,13), + (4941843,14,14), + (4941843,15,15), + (4941843,16,16), + (4941843,17,17), + (4941843,18,18), + (4941843,19,19), + (4941843,20,20), + (4941843,21,21), + (4941843,22,22), + (4941843,23,23), + (4941843,24,24), + (7147825,2,25646), + (7147825,3,3), + (7147825,4,4454), + (7147825,5,5), + (7147825,6,6), + (7147825,7,7), + (7147825,8,8), + (7147825,9,9), + (7147825,10,10), + (7147825,11,11), + (7147825,12,12), + (7147825,13,13), + (7147825,14,14), + (7147825,15,15), + (7147825,16,16), + (7147825,17,17), + (7147825,18,18), + (7147825,19,19), + (7147825,20,20), + (7147825,21,21), + (7147825,22,22), + (7147825,23,23), + (7147825,24,24), + (9462081,2,2), + (9462081,3,3544), + (9462081,4,4), + (9462081,5,564464), + (9462081,6,6), + (9462081,7,7), + (9462081,8,8), + (9462081,9,9), + (9462081,10,10), + (9462081,11,11), + (9462081,12,12), + (9462081,13,13), + (9462081,14,14), + (9462081,15,15), + (9462081,16,16), + (9462081,17,17), + (9462081,18,18), + (9462081,19,19), + (9462081,20,20), + (9462081,21,21), + (9462081,22,22), + (9462081,23,23), + (9462081,24,24), + (12935727,2,2), + (12935727,3,346465), + (12935727,4,46665), + (12935727,5,5), + (12935727,6,6), + (12935727,7,7), + (12935727,8,8), + (12935727,9,9), + (12935727,10,10), + (12935727,11,11), + (12935727,12,12), + (12935727,13,13), + (12935727,14,1454), + (12935727,15,15), + (12935727,16,16), + (12935727,17,17), + (12935727,18,18), + (12935727,19,19), + (12935727,20,20), + (12935727,21,21), + (12935727,22,22), + (12935727,23,23), + (12935727,24,24), + (14602384,2,2), + (14602384,3,36545), + (14602384,4,424), + (14602384,5,5454), + (14602384,6,654564), + (14602384,7,7), + (14602384,8,8), + (14602384,9,9), + (14602384,10,10), + (14602384,11,11), + (14602384,12,12), + (14602384,13,13), + (14602384,14,14), + (14602384,15,15), + (14602384,16,16), + (14602384,17,17), + (14602384,18,18), + (14602384,19,19), + (14602384,20,20), + (14602384,21,21), + (14602384,22,22), + (14602384,23,23), + (14602384,24,24), + (15121534,2,2564), + (15121534,3,35454), + (15121534,4,4), + (15121534,5,55132), + (15121534,6,6), + (15121534,7,7), + (15121534,8,8), + (15121534,9,9), + (15121534,10,10), + (15121534,11,11), + (15121534,12,12), + (15121534,13,13), + (15121534,14,14), + (15121534,15,15), + (15121534,16,16), + (15121534,17,17), + (15121534,18,18), + (15121534,19,19), + (15121534,20,20), + (15121534,21,21), + (15121534,22,22), + (15121534,23,23), + (15121534,24,24), + (1940168,2,21212), + (1940168,3,3), + (1940168,4,4), + (1940168,5,5), + (1940168,6,6), + (1940168,7,7), + (1940168,8,8), + (1940168,9,9), + (1940168,10,10), + (1940168,11,11), + (1940168,12,12), + (1940168,13,13), + (1940168,14,14), + (1940168,15,15), + (1940168,16,16), + (1940168,17,17), + (1940168,18,18), + (1940168,19,19), + (1940168,20,20), + (1940168,21,21), + (1940168,22,22), + (1940168,23,23), + (1940168,24,24), + (4941785,2,2), + (4941785,3,3), + (4941785,4,41221), + (4941785,5,5), + (4941785,6,6), + (4941785,7,7), + (4941785,8,8), + (4941785,9,9), + (4941785,10,10), + (4941785,11,11), + (4941785,12,12), + (4941785,13,13), + (4941785,14,14), + (4941785,15,15), + (4941785,16,16), + (4941785,17,17), + (4941785,18,18), + (4941785,19,19), + (4941785,20,20), + (4941785,21,21), + (4941785,22,22), + (4941785,23,23), + (4941785,24,24), + (7005058,2,2), + (7005058,3,3), + (7005058,4,4454), + (7005058,5,5), + (7005058,6,6), + (7005058,7,74454), + (7005058,8,8), + (7005058,9,9), + (7005058,10,10), + (7005058,11,11), + (7005058,12,12), + (7005058,13,13), + (7005058,14,14), + (7005058,15,15), + (7005058,16,16), + (7005058,17,17), + (7005058,18,18), + (7005058,19,19), + (7005058,20,20), + (7005058,21,21), + (7005058,22,22), + (7005058,23,23), + (7005058,24,24), + (9529321,2,2464), + (9529321,3,34554), + (9529321,4,4), + (9529321,5,5544), + (9529321,6,6), + (9529321,7,7), + (9529321,8,8), + (9529321,9,9), + (9529321,10,10), + (9529321,11,11), + (9529321,12,12), + (9529321,13,13), + (9529321,14,14), + (9529321,15,15), + (9529321,16,16), + (9529321,17,17), + (9529321,18,18), + (9529321,19,19), + (9529321,20,20), + (9529321,21,21), + (9529321,22,22), + (9529321,23,23), + (9529321,24,24), + (13210211,2,24465), + (13210211,3,34454), + (13210211,4,4), + (13210211,5,5), + (13210211,6,6), + (13210211,7,7), + (13210211,8,8), + (13210211,9,9), + (13210211,10,10), + (13210211,11,11), + (13210211,12,12), + (13210211,13,13), + (13210211,14,14), + (13210211,15,15), + (13210211,16,16), + (13210211,17,17), + (13210211,18,18), + (13210211,19,19), + (13210211,20,20), + (13210211,21,21), + (13210211,22,22), + (13210211,23,23), + (13210211,24,24), + (14377161,2,4545), + (14377161,3,3), + (14377161,4,4), + (14377161,5,5), + (14377161,6,656455), + (14377161,7,7), + (14377161,8,8), + (14377161,9,9), + (14377161,10,10), + (14377161,11,11), + (14377161,12,12), + (14377161,13,13), + (14377161,14,14), + (14377161,15,15), + (14377161,16,16), + (14377161,17,17), + (14377161,18,18), + (14377161,19,19), + (14377161,20,20), + (14377161,21,21), + (14377161,22,22), + (14377161,23,23), + (14377161,24,24), + (15114040,2,2), + (15114040,3,3), + (15114040,4,4), + (15114040,5,54655656), + (15114040,6,6445), + (15114040,7,7), + (15114040,8,8), + (15114040,9,9), + (15114040,10,10), + (15114040,11,11), + (15114040,12,12), + (15114040,13,13), + (15114040,14,14), + (15114040,15,15), + (15114040,16,16), + (15114040,17,17), + (15114040,18,18), + (15114040,19,19), + (15114040,20,20455), + (15114040,21,21), + (15114040,22,22), + (15114040,23,23), + (15114040,24,24) +; diff --git a/notes.md b/notes.md index 371cfee..9d5da08 100644 --- a/notes.md +++ b/notes.md @@ -1,26 +1,23 @@ +# kubectl +```sh +kubectl port-forward -n kafka svc/bd-prd-kafka-service 9094:9094 +kubectl port-forward -n database svc/mysql 3306:3306 +kubectl port-forward -n bd-prd svc/private-api-svc 5000:5000 +``` -# setup -## windows ```sh python -m venv .venv .venv\Scripts\activate python -m pip install --upgrade pip pip install -r requirements.txt ``` -## linux -```sh -python3 -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -pip install -r requirements.txt -``` ```sh .venv\Scripts\activate pip freeze > requirements.txt ``` -### tips linux / wsl +# linux / wsl to open vscode in wsl just open vs code, type `wsl` in the terminal than type `code .` tip: you can trim your command line path with @@ -31,9 +28,13 @@ add at the botom, exit nano with ctrl + x then press y to save ```sh PROMPT_DIRTRIM=3 ``` +restart your shell -# kubectl ```sh -kubectl port-forward -n kafka svc/bd-prd-kafka-service 9094:9094 -kubectl port-forward -n database svc/mysql 3306:3306 +sudo apt update -y && sudo apt upgrade -y +sudo apt install python3.10-venv -y ``` +```sh +python3 -m venv .venv +touch .venv\bin\activate +``` diff --git a/requirements.txt b/requirements.txt index 99accdd..492ae4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,34 +1,33 @@ -aiokafka==0.8.1 +aiokafka==0.10.0 aiomysql==0.2.0 -annotated-types==0.5.0 -anyio==3.7.1 +annotated-types==0.6.0 +anyio==4.3.0 async-timeout==4.0.3 -asyncmy==0.2.8 +asyncmy==0.2.9 cffi==1.16.0 click==8.1.7 colorama==0.4.6 -cryptography==41.0.4 -databases==0.8.0 -exceptiongroup==1.1.3 -fastapi==0.103.1 -greenlet==2.0.2 +cryptography==42.0.5 +databases==0.9.0 +exceptiongroup==1.2.0 +fastapi==0.110.0 +greenlet==3.0.3 h11==0.14.0 -httptools==0.6.0 -idna==3.4 +httptools==0.6.1 +idna==3.6 kafka-python==2.0.2 -packaging==23.1 +packaging==24.0 pycparser==2.21 -pydantic==2.3.0 -pydantic-settings==2.0.3 -pydantic_core==2.6.3 +pydantic==2.6.4 +pydantic-settings==2.2.1 +pydantic_core==2.16.3 PyMySQL==1.1.0 -python-dotenv==1.0.0 +python-dotenv==1.0.1 PyYAML==6.0.1 -sniffio==1.3.0 -SQLAlchemy==1.4.49 -starlette==0.27.0 -typing_extensions==4.7.1 -uvicorn==0.23.2 -watchfiles==0.20.0 -websockets==11.0.3 -cryptography==41.0.4 \ No newline at end of file +sniffio==1.3.1 +SQLAlchemy==2.0.28 +starlette==0.36.3 +typing_extensions==4.10.0 +uvicorn==0.29.0 +watchfiles==0.21.0 +websockets==12.0 diff --git a/src/api/__init__.py b/src/api/__init__.py index f940d62..3e8e7d0 100644 --- a/src/api/__init__.py +++ b/src/api/__init__.py @@ -1,5 +1,7 @@ from fastapi import APIRouter -from . import v2 + +from . import v2, v3 router = APIRouter() router.include_router(v2.router, prefix="/v2") +router.include_router(v3.router, prefix="/v3") diff --git a/src/api/v2/highscore.py b/src/api/v2/highscore.py index 510224e..75b9af3 100644 --- a/src/api/v2/highscore.py +++ b/src/api/v2/highscore.py @@ -1,5 +1,7 @@ -from fastapi import APIRouter, Query -from src.app.repositories.highscore import HighscoreLatest +from fastapi import APIRouter, Depends, Query + +from src.app.repositories.highscore import HighscoreRepo +from src.core.fastapi.dependencies.session import get_session router = APIRouter() @@ -10,28 +12,10 @@ async def get_highscore_latest( label_id: int = None, many: bool = False, limit: int = Query(default=10, ge=0, le=10_000), + session=Depends(get_session), ): - repo = HighscoreLatest() - if many: - data = await repo.get_many(start=player_id, limit=limit, label_id=label_id) - else: - data = await repo.get(id=player_id) + repo = HighscoreRepo(session=session) + data = await repo.select( + player_id=player_id, label_id=label_id, many=many, limit=limit + ) return data - - -# @router.get("/highscore") -# async def get_highscore( -# player_id: str = None, -# greater_than: bool = None, -# limit: int = Query(default=1_000, ge=0, le=10_000), -# ): -# return {} - - -# @router.get("/highscore/xp") -# async def get_highscore_xp( -# player_id: str = None, -# greater_than: bool = None, -# limit: int = Query(default=1_000, ge=0, le=10_000), -# ): -# return {} diff --git a/src/api/v2/player.py b/src/api/v2/player.py index 9c05475..743a58b 100644 --- a/src/api/v2/player.py +++ b/src/api/v2/player.py @@ -1,5 +1,7 @@ -from fastapi import APIRouter, Query -from src.app.repositories.player import Player +from fastapi import APIRouter, Depends, Query + +from src.app.repositories.player import PlayerRepo +from src.core.fastapi.dependencies.session import get_session router = APIRouter() @@ -9,13 +11,14 @@ async def get_player( player_id: str = None, player_name: str = None, label_id: int = None, - greater_than: bool = None, + greater_than: bool = False, limit: int = Query(default=1_000, ge=0, le=100_000), + session=Depends(get_session), ): # TODO: make use of abstract base class - repo = Player() + repo = PlayerRepo(session=session) - data = await repo.get_player( + data = await repo.select( player_id=player_id, player_name=player_name, greater_than=greater_than, diff --git a/src/api/v3/__init__.py b/src/api/v3/__init__.py new file mode 100644 index 0000000..8d6dbdc --- /dev/null +++ b/src/api/v3/__init__.py @@ -0,0 +1,6 @@ +from fastapi import APIRouter + +from . import highscore + +router = APIRouter() +router.include_router(highscore.router) diff --git a/src/api/v3/highscore.py b/src/api/v3/highscore.py new file mode 100644 index 0000000..1592bb7 --- /dev/null +++ b/src/api/v3/highscore.py @@ -0,0 +1,45 @@ +import logging + +from fastapi import APIRouter, Depends, Query + +from src.app.repositories import PlayerActivityRepo, PlayerSkillsRepo, ScraperDataRepo +from src.core.fastapi.dependencies.session import get_session + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/highscore/latest") +async def get_highscore_latest( + player_id: int, + player_name: str = None, + label_id: int = None, + many: bool = False, + limit: int = Query(default=10, ge=0, le=10_000), + session=Depends(get_session), +): + repo = ScraperDataRepo(session=session) + repo_skills = PlayerSkillsRepo(session=session) + repo_activities = PlayerActivityRepo(session=session) + + data = await repo.select( + player_name=player_name, + player_id=player_id, + label_id=label_id, + many=many, + limit=limit, + ) + + for d in data: + skills = await repo_skills.select(scraper_id=d.get("scraper_id")) + + d["skills"] = { + skill.get("skill_name"): skill.get("skill_value") for skill in skills + } + activities = await repo_activities.select(scraper_id=d.get("scraper_id")) + d["activity"] = { + activity.get("activity_name"): activity.get("activity_value") + for activity in activities + } + return data diff --git a/src/app/repositories/__init__.py b/src/app/repositories/__init__.py index e69de29..f55ad84 100644 --- a/src/app/repositories/__init__.py +++ b/src/app/repositories/__init__.py @@ -0,0 +1,15 @@ +from .abstract_repo import AbstractAPI +from .highscore import HighscoreRepo +from .player import PlayerRepo +from .player_activities import PlayerActivityRepo +from .player_skills import PlayerSkillsRepo +from .scraper_data import ScraperDataRepo + +__all__ = [ + "HighscoreRepo", + "PlayerRepo", + "PlayerSkillsRepo", + "ScraperDataRepo", + "AbstractAPI", + "PlayerActivityRepo", +] diff --git a/src/app/repositories/abstract_repo.py b/src/app/repositories/abstract_repo.py index fe94af2..ebc0ca4 100644 --- a/src/app/repositories/abstract_repo.py +++ b/src/app/repositories/abstract_repo.py @@ -1,18 +1,19 @@ from abc import ABC, abstractmethod + class AbstractAPI(ABC): @abstractmethod - def get(self, id): + def insert(self): raise NotImplementedError @abstractmethod - def get_many(self, start, limit): + def select(self): raise NotImplementedError @abstractmethod - def update(self, id, data): + def update(self): raise NotImplementedError @abstractmethod - def delete(self, id): + def delete(self): raise NotImplementedError diff --git a/src/app/repositories/highscore.py b/src/app/repositories/highscore.py index 0dbbc43..9dadedb 100644 --- a/src/app/repositories/highscore.py +++ b/src/app/repositories/highscore.py @@ -1,62 +1,55 @@ -from src.core.database.models.highscore import ( - # playerHiscoreData, +import logging + +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import ( # playerHiscoreData,; PlayerHiscoreDataXPChange, PlayerHiscoreDataLatest, - # PlayerHiscoreDataXPChange, ) from src.core.database.models.player import Player -from src.core.database.database import SessionFactory -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncResult -from sqlalchemy.sql.expression import Select -from fastapi.encoders import jsonable_encoder -from src.app.repositories.abstract_repo import AbstractAPI + +logger = logging.getLogger(__name__) -class HighscoreLatest(AbstractAPI): - def __init__(self) -> None: +class HighscoreRepo(AbstractAPI): + def __init__(self, session) -> None: super().__init__() - self.table = PlayerHiscoreDataLatest - - async def _simple_execute(self, sql) -> dict: - async with SessionFactory() as session: - session: AsyncSession - - result: AsyncResult = await session.execute(sql) - result = result.all() - return jsonable_encoder(result) - - async def get(self, id: int): - sql: Select = select(self.table, Player.name) - sql = sql.join(target=Player, onclause=self.table.Player_id == Player.id) - sql = sql.where(self.table.Player_id == id) - data: list[dict] = await self._simple_execute(sql) - data = [{"name": d.pop("name"), **d["PlayerHiscoreDataLatest"]} for d in data] - return data + self.session: AsyncSession = session + + def insert(self): + raise NotImplementedError + + async def select( + self, player_id: int, label_id: int, limit: int, many: bool + ) -> dict: + table = aliased(PlayerHiscoreDataLatest, name="phd") + player = aliased(Player, name="pl") + + sql = Select(player.name, table) + sql = sql.join(target=player, onclause=table.Player_id == player.id) - async def get_many( - self, - start: int, - label_id: int = None, - limit: int = 5000, - ): - sql: Select = select(self.table, Player.name) - sql = sql.join(target=Player, onclause=self.table.Player_id == Player.id) - sql = sql.where(self.table.Player_id > start) + if player_id: + if many: + sql = sql.where(table.Player_id >= player_id) + else: + sql = sql.where(table.Player_id == player_id) if label_id: - sql = sql.where(Player.label_id == label_id) + sql = sql.where(player.label_id == label_id) sql = sql.limit(limit) - sql = sql.order_by(self.table.Player_id.asc()) - data: list[dict] = await self._simple_execute(sql) - # data = [{"PlayerHiscoreDataLatest":{"total": int, ...}, "name": str}] - data = [{"name": d.pop("name"), **d["PlayerHiscoreDataLatest"]} for d in data] + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"name": name, **jsonable_encoder(hs)} for name, hs in result] return data - async def delete(self, id): - pass + async def update(self): + raise NotImplementedError - async def update(self, id, data): - pass + async def delete(self): + raise NotImplementedError diff --git a/src/app/repositories/player.py b/src/app/repositories/player.py index 652f649..32e0823 100644 --- a/src/app/repositories/player.py +++ b/src/app/repositories/player.py @@ -1,17 +1,15 @@ -from src.core.database.models.player import Player as dbPlayer -from src.core.database.database import SessionFactory -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncResult -from sqlalchemy.sql.expression import Select from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.sql.expression import Select + +from src.core.database.models.player import Player -class Player: - def __init__(self) -> None: - pass +class PlayerRepo: + def __init__(self, session: AsyncSession) -> None: + self.session = session - async def get_player( + async def select( self, player_id: int, player_name: str, @@ -19,25 +17,25 @@ async def get_player( greater_than: bool, limit: int = 1_000, ): - table = dbPlayer - sql: Select = select(table) - sql = sql.limit(limit) + table = Player + sql = Select(table) if player_name: - sql = sql.where(dbPlayer.name >= player_name) + sql = sql.where(table.name == player_name) if label_id: - sql = sql.where(dbPlayer.label_id == label_id) + sql = sql.where(table.label_id == label_id) - comparison = ( - (dbPlayer.id >= player_id) if greater_than else (dbPlayer.id == player_id) - ) - sql = sql.where(comparison) - sql = sql.order_by(dbPlayer.id.asc()) + if player_id: + if greater_than: + sql = sql.where(table.id >= player_id) + else: + sql = sql.where(table.id == player_id) - async with SessionFactory() as session: - session: AsyncSession + sql = sql.order_by(table.id.asc()) + sql = sql.limit(limit) - result: AsyncResult = await session.execute(sql) + async with self.session: + result: AsyncResult = await self.session.execute(sql) result = result.scalars().all() return jsonable_encoder(result) diff --git a/src/app/repositories/player_activities.py b/src/app/repositories/player_activities.py new file mode 100644 index 0000000..b6c0993 --- /dev/null +++ b/src/app/repositories/player_activities.py @@ -0,0 +1,49 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import Activities, PlayerActivities + + +class PlayerActivityRepo(AbstractAPI): + def __init__(self, session: AsyncSession) -> None: + super().__init__() + self.session = session + + async def insert(self, id): + raise NotImplementedError + + async def select( + self, + scraper_id: int = None, + activity_id: int = None, + limit: int = None, + ): + table = aliased(PlayerActivities, name="pa") + + sql = Select(Activities.activity_name, table) + + if scraper_id: + sql = sql.where(table.scraper_id == scraper_id) + + if activity_id: + sql = sql.where(table.activity_id == activity_id) + + if limit: + sql = sql.limit(limit) + + sql = sql.join(Activities, table.activity_id == Activities.activity_id) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"activity_name": name, **jsonable_encoder(hs)} for name, hs in result] + return data + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/src/app/repositories/player_skills.py b/src/app/repositories/player_skills.py new file mode 100644 index 0000000..ea9705d --- /dev/null +++ b/src/app/repositories/player_skills.py @@ -0,0 +1,49 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import PlayerSkills, Skills + + +class PlayerSkillsRepo(AbstractAPI): + def __init__(self, session: AsyncSession) -> None: + super().__init__() + self.session = session + + async def insert(self, id): + raise NotImplementedError + + async def select( + self, + scraper_id: int = None, + skill_id: int = None, + limit: int = None, + ): + table = aliased(PlayerSkills, name="ps") + + sql = Select(Skills.skill_name, table) + + if scraper_id: + sql = sql.where(table.scraper_id == scraper_id) + + if skill_id: + sql = sql.where(table.skill_id == skill_id) + + if limit: + sql = sql.limit(limit) + + sql = sql.join(Skills, table.skill_id == Skills.skill_id) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.fetchall() + data = [{"skill_name": name, **jsonable_encoder(hs)} for name, hs in result] + return data + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/src/app/repositories/scraper_data.py b/src/app/repositories/scraper_data.py new file mode 100644 index 0000000..97515d9 --- /dev/null +++ b/src/app/repositories/scraper_data.py @@ -0,0 +1,79 @@ +from fastapi.encoders import jsonable_encoder +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +from src.app.repositories.abstract_repo import AbstractAPI +from src.core.database.models import Player, ScraperData, ScraperDataLatest + + +class ScraperDataRepo(AbstractAPI): + def __init__(self, session: AsyncSession) -> None: + super().__init__() + self.session = session + + async def insert(self, id): + raise NotImplementedError + + async def select( + self, + player_name: str, + player_id: int, + label_id: int, + many: bool, + limit: int, + history: bool = False, + ) -> list[dict]: + table = ( + aliased(ScraperData, name="sd") + if history + else aliased(ScraperDataLatest, name="sdl") + ) + player = aliased(Player, name="pl") + + sql = Select(table) + sql = sql.join(player, table.player_id == player.id) + + if player_id: + if many: + sql = sql.where(table.player_id >= player_id) + else: + sql = sql.where(table.player_id == player_id) + + if player_name: + sql = sql.where(player.name == player_name) + + if label_id: + sql = sql.where(player.label_id == label_id) + + sql = sql.limit(limit) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.scalars().all() + return jsonable_encoder(result) + + async def select_history(self, player_name: str, player_id: int, many: bool): + table = ScraperData + sql = Select(table) + + if player_id: + if many: + sql = sql.where(table.player_id >= player_id) + else: + sql = sql.where(table.player_id == player_id) + + if player_name: + sql = sql.join(Player, table.player_id == Player.id) + sql = sql.where(Player.name == player_name) + + async with self.session: + result: AsyncResult = await self.session.execute(sql) + result = result.scalars().all() + return jsonable_encoder(result) + + async def update(self): + raise NotImplementedError + + async def delete(self): + raise NotImplementedError diff --git a/src/core/.gitkeep b/src/core/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/core/__init__.py b/src/core/__init__.py index 384d4b0..574e129 100644 --- a/src/core/__init__.py +++ b/src/core/__init__.py @@ -1 +1,4 @@ -# needed for log formatting \ No newline at end of file +# needed for log formatting +from . import logging_config + +__all__ = ["logging_config"] diff --git a/src/core/database/database.py b/src/core/database/database.py index dd7c750..b02fb3f 100644 --- a/src/core/database/database.py +++ b/src/core/database/database.py @@ -1,6 +1,5 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import declarative_base, sessionmaker from src.core.config import settings diff --git a/src/core/database/models/__init__.py b/src/core/database/models/__init__.py new file mode 100644 index 0000000..58d7b55 --- /dev/null +++ b/src/core/database/models/__init__.py @@ -0,0 +1,26 @@ +from .activities import Activities, PlayerActivities +from .highscore import ( + PlayerHiscoreDataLatest, + PlayerHiscoreDataXPChange, + playerHiscoreData, +) +from .player import Player +from .prediction import Prediction +from .report import Report +from .scraper_data import ScraperData, ScraperDataLatest +from .skills import PlayerSkills, Skills + +__all__ = [ + "Activities", + "PlayerActivities", + "playerHiscoreData", + "PlayerHiscoreDataLatest", + "PlayerHiscoreDataXPChange", + "Player", + "Prediction", + "Report", + "ScraperData", + "ScraperDataLatest", + "PlayerSkills", + "Skills", +] diff --git a/src/core/database/models/activities.py b/src/core/database/models/activities.py new file mode 100644 index 0000000..e252c72 --- /dev/null +++ b/src/core/database/models/activities.py @@ -0,0 +1,27 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.dialects.mysql import BIGINT, TINYINT + +from src.core.database.database import Base + + +class Activities(Base): + __tablename__ = "activities" + + activity_id = Column(TINYINT, primary_key=True, autoincrement=True) + activity_name = Column(String(50), nullable=False) + + +class PlayerActivities(Base): + __tablename__ = "player_activities" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_id = Column( + TINYINT, + ForeignKey("activities.activity_id", ondelete="CASCADE"), + primary_key=True, + ) + activity_value = Column(Integer, nullable=False, default=0) diff --git a/src/core/database/models/highscore.py b/src/core/database/models/highscore.py index b5e33f4..e28bcb4 100644 --- a/src/core/database/models/highscore.py +++ b/src/core/database/models/highscore.py @@ -9,12 +9,9 @@ text, ) -from sqlalchemy.ext.declarative import declarative_base +from src.core.database.database import Base -Base = declarative_base() -metadata = Base.metadata - class playerHiscoreData(Base): __tablename__ = "playerHiscoreData" __table_args__ = ( @@ -240,6 +237,8 @@ class PlayerHiscoreDataLatest(Base): the_leviathan = Column(Integer, default=0) the_whisperer = Column(Integer, default=0) vardorvis = Column(Integer, default=0) + + class PlayerHiscoreDataXPChange(Base): __tablename__ = "playerHiscoreDataXPChange" @@ -348,4 +347,4 @@ class PlayerHiscoreDataXPChange(Base): spindel = Column(Integer, default=0) the_leviathan = Column(Integer, default=0) the_whisperer = Column(Integer, default=0) - vardorvis = Column(Integer, default=0) \ No newline at end of file + vardorvis = Column(Integer, default=0) diff --git a/src/core/database/models/scraper_data.py b/src/core/database/models/scraper_data.py new file mode 100644 index 0000000..e14021a --- /dev/null +++ b/src/core/database/models/scraper_data.py @@ -0,0 +1,22 @@ +from sqlalchemy import Column, Date, DateTime, func +from sqlalchemy.dialects.mysql import BIGINT, SMALLINT + +from src.core.database.database import Base + + +class ScraperData(Base): + __tablename__ = "scraper_data" + + scraper_id = Column(BIGINT, primary_key=True, autoincrement=True) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(SMALLINT, nullable=False) + record_date = Column(Date, nullable=True) + + +class ScraperDataLatest(Base): + __tablename__ = "scraper_data_latest" + + scraper_id = Column(BIGINT) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + player_id = Column(BIGINT, primary_key=True) + record_date = Column(Date, nullable=True) diff --git a/src/core/database/models/skills.py b/src/core/database/models/skills.py new file mode 100644 index 0000000..a7e4080 --- /dev/null +++ b/src/core/database/models/skills.py @@ -0,0 +1,27 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.dialects.mysql import BIGINT, TINYINT + +from src.core.database.database import Base + + +class Skills(Base): + __tablename__ = "skills" + + skill_id = Column(TINYINT, primary_key=True, autoincrement=True) + skill_name = Column(String(50), nullable=False) + + +class PlayerSkills(Base): + __tablename__ = "player_skills" + + scraper_id = Column( + BIGINT, + ForeignKey("scraper_data.scraper_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_id = Column( + TINYINT, + ForeignKey("skills.skill_id", ondelete="CASCADE"), + primary_key=True, + ) + skill_value = Column(Integer, nullable=False, default=0) diff --git a/src/core/logging.py b/src/core/logging_config.py similarity index 80% rename from src/core/logging.py rename to src/core/logging_config.py index 46f8f95..92581f2 100644 --- a/src/core/logging.py +++ b/src/core/logging_config.py @@ -2,7 +2,6 @@ import logging import sys - # # log formatting formatter = logging.Formatter( json.dumps( @@ -25,10 +24,10 @@ logging.basicConfig(level=logging.DEBUG, handlers=handlers) # set imported loggers to warning -logging.getLogger("urllib3").setLevel(logging.DEBUG) -logging.getLogger("uvicorn").setLevel(logging.DEBUG) -logging.getLogger("aiomysql").setLevel(logging.ERROR) -logging.getLogger("aiokafka").setLevel(logging.WARNING) +# logging.getLogger("urllib3").setLevel(logging.DEBUG) +# logging.getLogger("uvicorn").setLevel(logging.DEBUG) +# logging.getLogger("aiomysql").setLevel(logging.ERROR) +# logging.getLogger("aiokafka").setLevel(logging.WARNING) # if settings.ENV == "PRD": # uvicorn_error = logging.getLogger("uvicorn.error") diff --git a/src/core/server.py b/src/core/server.py index 8abe404..dc20696 100644 --- a/src/core/server.py +++ b/src/core/server.py @@ -7,8 +7,6 @@ from src import api from src.core.fastapi.middleware.logging import LoggingMiddleware - - logger = logging.getLogger(__name__) @@ -30,7 +28,7 @@ def make_middleware() -> list[Middleware]: allow_methods=["*"], allow_headers=["*"], ), - Middleware(LoggingMiddleware) + Middleware(LoggingMiddleware), ] return middleware @@ -51,13 +49,3 @@ def create_app() -> FastAPI: @app.get("/") async def root(): return {"message": "Hello World"} - - -@app.on_event("startup") -async def startup_event(): - pass - - -@app.on_event("shutdown") -async def shutdown_event(): - pass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..92be3c9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,58 @@ +# conftest.py +import os +import sys +from contextlib import asynccontextmanager + +import pytest +from fastapi import FastAPI +from httpx import AsyncClient +from httpx._transports.asgi import ASGITransport +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from src.core import server # noqa: E402 +from src.core.fastapi.dependencies.session import get_session # noqa: E402 + +# Create an async SQLAlchemy engine +engine = create_async_engine( + "mysql+aiomysql://root:root_bot_buster@localhost:3307/playerdata", + pool_timeout=30, + pool_recycle=30, + echo=True, + pool_pre_ping=True, +) + +# Create a session factory +SessionFactory = sessionmaker( + bind=engine, + expire_on_commit=False, + class_=AsyncSession, # Use AsyncSession for asynchronous operations +) + + +async def get_session_override(): + async with SessionFactory() as session: + session: AsyncSession + yield session + await engine.dispose() + return + + +server.app.dependency_overrides[get_session] = get_session_override + + +@pytest.fixture +def app() -> FastAPI: + return server.app + + +@pytest.fixture +@asynccontextmanager +async def custom_client(app: FastAPI): + base_url = "http://srv.test/" + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url=base_url) as client: + yield client diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..4088045 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode=auto diff --git a/tests/test_highscore.py b/tests/test_highscore.py new file mode 100644 index 0000000..4a1301f --- /dev/null +++ b/tests/test_highscore.py @@ -0,0 +1,44 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_hs_id_v2(custom_client): + endpoint = "/v2/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("id") == 1, f"expected id: 1 got: {player=}" + + +@pytest.mark.asyncio +async def test_one_hs_id_v3(custom_client): + endpoint = "/v3/highscore/latest" + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance( + json_response[0], dict + ), f"expected dict, got {type(json_response[0])}, {json_response=}" + + player = json_response[0] + assert player.get("player_id") == 1, f"expected id: 1 got: {player=}" diff --git a/tests/test_player.py b/tests/test_player.py new file mode 100644 index 0000000..53eb292 --- /dev/null +++ b/tests/test_player.py @@ -0,0 +1,76 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.asyncio +async def test_one_player_id(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1, "greater_than": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("id") == 1 + + +@pytest.mark.asyncio +async def test_one_player_name(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_name": "player1", "greater_than": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) == 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("id") == 9 + + +@pytest.mark.asyncio +async def test_many_player(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"player_id": 1, "greater_than": 1} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) > 1 + assert isinstance(json_response[0], dict) + player = json_response[0] + assert player.get("id") == 1 + + +@pytest.mark.asyncio +async def test_player_label(custom_client): + endpoint = "/v2/player" + + async with custom_client as client: + client: AsyncClient + params = {"label_id": 0} + response = await client.get(url=endpoint, params=params) + + assert response.status_code == 200 + assert isinstance(response.json(), list) + + json_response: list[dict] = response.json() + assert len(json_response) >= 1 + assert isinstance(json_response[0], dict)