From d5d3dd4157efc5f3082c0dd2796de59ac8a1cd64 Mon Sep 17 00:00:00 2001
From: extreme4all <>
Date: Sat, 20 Apr 2024 11:45:17 +0200
Subject: [PATCH] performance improvements & added name

---
 src/api/v2/highscore.py              |   1 +
 src/app/repositories/scraper_data.py | 162 +++++----
 src/app/views/response/highscore.py  |   1 +
 tests/test_highscore_v2.py           | 525 ++++++++++++++-------------
 4 files changed, 347 insertions(+), 342 deletions(-)

diff --git a/src/api/v2/highscore.py b/src/api/v2/highscore.py
index dd14868..8a509ae 100644
--- a/src/api/v2/highscore.py
+++ b/src/api/v2/highscore.py
@@ -53,6 +53,7 @@ async def get_highscore_latest_v2(
         history=False,
     )
 
+    logger.info(data[0])
     for d in data:
         scraper_id = d.pop("scraper_id")
         d["Player_id"] = d.pop("player_id")
diff --git a/src/app/repositories/scraper_data.py b/src/app/repositories/scraper_data.py
index 3394cb4..c16413d 100644
--- a/src/app/repositories/scraper_data.py
+++ b/src/app/repositories/scraper_data.py
@@ -1,80 +1,82 @@
-from fastapi.encoders import jsonable_encoder
-from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession
-from sqlalchemy.orm import aliased
-from sqlalchemy.sql.expression import Select
-
-from src.app.repositories.abstract_repo import AbstractAPI
-from src.core.database.models import Player, ScraperData, ScraperDataLatest
-
-
-class ScraperDataRepo(AbstractAPI):
-    def __init__(self, session: AsyncSession) -> None:
-        super().__init__()
-        self.session = session
-
-    async def insert(self, id):
-        raise NotImplementedError
-
-    async def select(
-        self,
-        player_name: str,
-        player_id: int,
-        label_id: int,
-        many: bool,
-        limit: int,
-        history: bool = False,
-    ) -> list[dict]:
-        table = (
-            aliased(ScraperData, name="sd")
-            if history
-            else aliased(ScraperDataLatest, name="sdl")
-        )
-        player = aliased(Player, name="pl")
-
-        sql = Select(table)
-        sql = sql.join(player, table.player_id == player.id)
-
-        if player_id:
-            if many:
-                sql = sql.where(table.player_id >= player_id)
-            else:
-                sql = sql.where(table.player_id == player_id)
-
-        if player_name:
-            sql = sql.where(player.name == player_name)
-
-        if label_id:
-            sql = sql.where(player.label_id == label_id)
-
-        sql = sql.order_by(player.id.asc())
-        sql = sql.limit(limit)
-
-        async with self.session:
-            result: AsyncResult = await self.session.execute(sql)
-            result = result.scalars().all()
-        return jsonable_encoder(result)
-
-    async def select_history(self, player_name: str, player_id: int, many: bool):
-        table = ScraperData
-        sql = Select(table)
-
-        if player_id:
-            if many:
-                sql = sql.where(table.player_id >= player_id)
-            else:
-                sql = sql.where(table.player_id == player_id)
-
-        if player_name:
-            sql = sql.join(Player, table.player_id == Player.id)
-            sql = sql.where(Player.name == player_name)
-
-        async with self.session:
-            result: AsyncResult = await self.session.execute(sql)
-            result = result.scalars().all()
-        return jsonable_encoder(result)
-
-    async def update(self):
-        raise NotImplementedError
-
-    async def delete(self):
-        raise NotImplementedError
+from fastapi.encoders import jsonable_encoder
+from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession
+from sqlalchemy.orm import aliased
+from sqlalchemy.sql.expression import Select
+
+from src.app.repositories.abstract_repo import AbstractAPI
+from src.core.database.models import Player, ScraperData, ScraperDataLatest
+
+
+class ScraperDataRepo(AbstractAPI):
+    def __init__(self, session: AsyncSession) -> None:
+        super().__init__()
+        self.session = session
+
+    async def insert(self, id):
+        raise NotImplementedError
+
+    async def select(
+        self,
+        player_name: str,
+        player_id: int,
+        label_id: int,
+        many: bool,
+        limit: int,
+        history: bool = False,
+    ) -> list[dict]:
+        table = (
+            aliased(ScraperData, name="sd")
+            if history
+            else aliased(ScraperDataLatest, name="sdl")
+        )
+        player = aliased(Player, name="pl")
+
+        sql = Select(player.name, table)
+        sql = sql.join(player, table.player_id == player.id)
+
+        if player_id:
+            if many:
+                sql = sql.where(table.player_id >= player_id)
+            else:
+                sql = sql.where(table.player_id == player_id)
+
+        if player_name:
+            sql = sql.where(player.name == player_name)
+
+        if label_id:
+            sql = sql.where(player.label_id == label_id)
+
+        sql = sql.order_by(table.player_id.asc())
+        # sql = sql.order_by(player.id.asc()) # not performant
+        sql = sql.limit(limit)
+
+        async with self.session:
+            result: AsyncResult = await self.session.execute(sql)
+            result = result.fetchall()
+        data = [{"name": name, **jsonable_encoder(r)} for name, r in result]
+        return data
+
+    async def select_history(self, player_name: str, player_id: int, many: bool):
+        table = ScraperData
+        sql = Select(table)
+
+        if player_id:
+            if many:
+                sql = sql.where(table.player_id >= player_id)
+            else:
+                sql = sql.where(table.player_id == player_id)
+
+        if player_name:
+            sql = sql.join(Player, table.player_id == Player.id)
+            sql = sql.where(Player.name == player_name)
+
+        async with self.session:
+            result: AsyncResult = await self.session.execute(sql)
+            result = result.scalars().all()
+        return jsonable_encoder(result)
+
+    async def update(self):
+        raise NotImplementedError
+
+    async def delete(self):
+        raise NotImplementedError
diff --git a/src/app/views/response/highscore.py b/src/app/views/response/highscore.py
index 0ef0e4b..2c475b9 100644
--- a/src/app/views/response/highscore.py
+++ b/src/app/views/response/highscore.py
@@ -10,6 +10,7 @@ class PlayerHiscoreData(BaseModel):
     id: Optional[int] = None
     timestamp: datetime = None
     ts_date: Optional[date] = None
+    name: str
     Player_id: int
     total: int = 0
     attack: int = 0
diff --git a/tests/test_highscore_v2.py b/tests/test_highscore_v2.py
index 7014ee9..f2c0f84 100644
--- a/tests/test_highscore_v2.py
+++ b/tests/test_highscore_v2.py
@@ -1,262 +1,263 @@
-import pytest
-from httpx import AsyncClient
-
-
-@pytest.mark.asyncio
-async def test_one_hs_id_v2(custom_client):
-    endpoint = "/v2/highscore/latest"
-    async with custom_client as client:
-        client: AsyncClient
-        params = {"player_id": 1}
-        response = await client.get(url=endpoint, params=params)
-
-        assert response.status_code == 200
-        assert isinstance(response.json(), list)
-
-        json_response: list[dict] = response.json()
-        assert len(json_response) == 1
-        assert isinstance(
-            json_response[0], dict
-        ), f"expected dict, got {type(json_response[0])}, {json_response=}"
-
-        player = json_response[0]
-        assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}"
-
-
-@pytest.mark.asyncio
-async def test_highscore_latest_v2(custom_client):
-    endpoint = "/v2/highscore/latest"
-    async with custom_client as client:
-        client: AsyncClient
-        params = {"player_id": 1}
-        response = await client.get(url=endpoint, params=params)
-
-        assert response.status_code == 200
-        assert isinstance(response.json(), list)
-
-        json_response: list[dict] = response.json()
-        assert len(json_response) == 1
-        assert isinstance(
-            json_response[0], dict
-        ), f"expected dict, got {type(json_response[0])}, {json_response=}"
-
-        player = json_response[0]
-        # assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}"
-
-        # List of keys that should be present in the player dictionary
-        keys = [
-            "id",
-            "timestamp",
-            "ts_date",
-            "Player_id",
-            "total",
-            "attack",
-            "defence",
-            "strength",
-            "hitpoints",
-            "ranged",
-            "prayer",
-            "magic",
-            "cooking",
-            "woodcutting",
-            "fletching",
-            "fishing",
-            "firemaking",
-            "crafting",
-            "smithing",
-            "mining",
-            "herblore",
-            "agility",
-            "thieving",
-            "slayer",
-            "farming",
-            "runecraft",
-            "hunter",
-            "construction",
-            "league",
-            "bounty_hunter_hunter",
-            "bounty_hunter_rogue",
-            "cs_all",
-            "cs_beginner",
-            "cs_easy",
-            "cs_medium",
-            "cs_hard",
-            "cs_elite",
-            "cs_master",
-            "lms_rank",
-            "soul_wars_zeal",
-            "abyssal_sire",
-            "alchemical_hydra",
-            "barrows_chests",
-            "bryophyta",
-            "callisto",
-            "cerberus",
-            "chambers_of_xeric",
-            "chambers_of_xeric_challenge_mode",
-            "chaos_elemental",
-            "chaos_fanatic",
-            "commander_zilyana",
-            "corporeal_beast",
-            "crazy_archaeologist",
-            "dagannoth_prime",
-            "dagannoth_rex",
-            "dagannoth_supreme",
-            "deranged_archaeologist",
-            "general_graardor",
-            "giant_mole",
-            "grotesque_guardians",
-            "hespori",
-            "kalphite_queen",
-            "king_black_dragon",
-            "kraken",
-            "kreearra",
-            "kril_tsutsaroth",
-            "mimic",
-            "nex",
-            "nightmare",
-            "phosanis_nightmare",
-            "obor",
-            "phantom_muspah",
-            "sarachnis",
-            "scorpia",
-            "skotizo",
-            "tempoross",
-            "the_gauntlet",
-            "the_corrupted_gauntlet",
-            "theatre_of_blood",
-            "theatre_of_blood_hard",
-            "thermonuclear_smoke_devil",
-            "tombs_of_amascut",
-            "tombs_of_amascut_expert",
-            "tzkal_zuk",
-            "tztok_jad",
-            "venenatis",
-            "vetion",
-            "vorkath",
-            "wintertodt",
-            "zalcano",
-            "zulrah",
-            "rifts_closed",
-            "artio",
-            "calvarion",
-            "duke_sucellus",
-            "spindel",
-            "the_leviathan",
-            "the_whisperer",
-            "vardorvis",
-        ]
-
-        # Check if all keys are present in the player dictionary
-        for key in keys:
-            assert key in player, f"Key {key} not found in player dictionary"
-
-        # Dictionary with expected types
-        expected_types = {
-            "id": int,
-            "timestamp": str,
-            "ts_date": str,
-            "Player_id": int,
-            "total": int,
-            "attack": int,
-            "defence": int,
-            "strength": int,
-            "hitpoints": int,
-            "ranged": int,
-            "prayer": int,
-            "magic": int,
-            "cooking": int,
-            "woodcutting": int,
-            "fletching": int,
-            "fishing": int,
-            "firemaking": int,
-            "crafting": int,
-            "smithing": int,
-            "mining": int,
-            "herblore": int,
-            "agility": int,
-            "thieving": int,
-            "slayer": int,
-            "farming": int,
-            "runecraft": int,
-            "hunter": int,
-            "construction": int,
-            "league": int,
-            "bounty_hunter_hunter": int,
-            "bounty_hunter_rogue": int,
-            "cs_all": int,
-            "cs_beginner": int,
-            "cs_easy": int,
-            "cs_medium": int,
-            "cs_hard": int,
-            "cs_elite": int,
-            "cs_master": int,
-            "lms_rank": int,
-            "soul_wars_zeal": int,
-            "abyssal_sire": int,
-            "alchemical_hydra": int,
-            "barrows_chests": int,
-            "bryophyta": int,
-            "callisto": int,
-            "cerberus": int,
-            "chambers_of_xeric": int,
-            "chambers_of_xeric_challenge_mode": int,
-            "chaos_elemental": int,
-            "chaos_fanatic": int,
-            "commander_zilyana": int,
-            "corporeal_beast": int,
-            "crazy_archaeologist": int,
-            "dagannoth_prime": int,
-            "dagannoth_rex": int,
-            "dagannoth_supreme": int,
-            "deranged_archaeologist": int,
-            "general_graardor": int,
-            "giant_mole": int,
-            "grotesque_guardians": int,
-            "hespori": int,
-            "kalphite_queen": int,
-            "king_black_dragon": int,
-            "kraken": int,
-            "kreearra": int,
-            "kril_tsutsaroth": int,
-            "mimic": int,
-            "nex": int,
-            "nightmare": int,
-            "phosanis_nightmare": int,
-            "obor": int,
-            "phantom_muspah": int,
-            "sarachnis": int,
-            "scorpia": int,
-            "skotizo": int,
-            "tempoross": int,
-            "the_gauntlet": int,
-            "the_corrupted_gauntlet": int,
-            "theatre_of_blood": int,
-            "theatre_of_blood_hard": int,
-            "thermonuclear_smoke_devil": int,
-            "tombs_of_amascut": int,
-            "tombs_of_amascut_expert": int,
-            "tzkal_zuk": int,
-            "tztok_jad": int,
-            "venenatis": int,
-            "vetion": int,
-            "vorkath": int,
-            "wintertodt": int,
-            "zalcano": int,
-            "zulrah": int,
-            "rifts_closed": int,
-            "artio": int,
-            "calvarion": int,
-            "duke_sucellus": int,
-            "spindel": int,
-            "the_leviathan": int,
-            "the_whisperer": int,
-            "vardorvis": int,
-        }
-
-        # Check if the type of each value in the returned player dictionary matches the expected type
-        for key, expected_type in expected_types.items():
-            value = player.get(key)
-            # if value is not None:
-            assert isinstance(
-                value, expected_type
-            ), f"Key {key} has incorrect type. Expected: {expected_type}, Got: {type(value)}"
+import pytest
+from httpx import AsyncClient
+
+
+@pytest.mark.asyncio
+async def test_one_hs_id_v2(custom_client):
+    endpoint = "/v2/highscore/latest"
+    async with custom_client as client:
+        client: AsyncClient
+        params = {"player_id": 1}
+        response = await client.get(url=endpoint, params=params)
+
+        assert response.status_code == 200
+        assert isinstance(response.json(), list)
+
+        json_response: list[dict] = response.json()
+        assert len(json_response) == 1
+        assert isinstance(
+            json_response[0], dict
+        ), f"expected dict, got {type(json_response[0])}, {json_response=}"
+
+        player = json_response[0]
+        assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}"
+
+
+@pytest.mark.asyncio
+async def test_highscore_latest_v2(custom_client):
+    endpoint = "/v2/highscore/latest"
+    async with custom_client as client:
+        client: AsyncClient
+        params = {"player_id": 1}
+        response = await client.get(url=endpoint, params=params)
+
+        assert response.status_code == 200
+        assert isinstance(response.json(), list)
+
+        json_response: list[dict] = response.json()
+        assert len(json_response) == 1
+        assert isinstance(
+            json_response[0], dict
+        ), f"expected dict, got {type(json_response[0])}, {json_response=}"
+
+        player = json_response[0]
+        # assert player.get("Player_id") == 1, f"expected Player_id: 1 got: {player=}"
+
+        # List of keys that should be present in the player dictionary
+        keys = [
+            "id",
+            "timestamp",
+            "ts_date",
+            "Player_id",
+            "name",
+            "total",
+            "attack",
+            "defence",
+            "strength",
+            "hitpoints",
+            "ranged",
+            "prayer",
+            "magic",
+            "cooking",
+            "woodcutting",
+            "fletching",
+            "fishing",
+            "firemaking",
+            "crafting",
+            "smithing",
+            "mining",
+            "herblore",
+            "agility",
+            "thieving",
+            "slayer",
+            "farming",
+            "runecraft",
+            "hunter",
+            "construction",
+            "league",
+            "bounty_hunter_hunter",
+            "bounty_hunter_rogue",
+            "cs_all",
+            "cs_beginner",
+            "cs_easy",
+            "cs_medium",
+            "cs_hard",
+            "cs_elite",
+            "cs_master",
+            "lms_rank",
+            "soul_wars_zeal",
+            "abyssal_sire",
+            "alchemical_hydra",
+            "barrows_chests",
+            "bryophyta",
+            "callisto",
+            "cerberus",
+            "chambers_of_xeric",
+            "chambers_of_xeric_challenge_mode",
+            "chaos_elemental",
+            "chaos_fanatic",
+            "commander_zilyana",
+            "corporeal_beast",
+            "crazy_archaeologist",
+            "dagannoth_prime",
+            "dagannoth_rex",
+            "dagannoth_supreme",
+            "deranged_archaeologist",
+            "general_graardor",
+            "giant_mole",
+            "grotesque_guardians",
+            "hespori",
+            "kalphite_queen",
+            "king_black_dragon",
+            "kraken",
+            "kreearra",
+            "kril_tsutsaroth",
+            "mimic",
+            "nex",
+            "nightmare",
+            "phosanis_nightmare",
+            "obor",
+            "phantom_muspah",
+            "sarachnis",
+            "scorpia",
+            "skotizo",
+            "tempoross",
+            "the_gauntlet",
+            "the_corrupted_gauntlet",
+            "theatre_of_blood",
+            "theatre_of_blood_hard",
+            "thermonuclear_smoke_devil",
+            "tombs_of_amascut",
+            "tombs_of_amascut_expert",
+            "tzkal_zuk",
+            "tztok_jad",
+            "venenatis",
+            "vetion",
+            "vorkath",
+            "wintertodt",
+            "zalcano",
+            "zulrah",
+            "rifts_closed",
+            "artio",
+            "calvarion",
+            "duke_sucellus",
+            "spindel",
+            "the_leviathan",
+            "the_whisperer",
+            "vardorvis",
+        ]
+
+        # Check if all keys are present in the player dictionary
+        for key in keys:
+            assert key in player, f"Key {key} not found in player dictionary"
+
+        # Dictionary with expected types
+        expected_types = {
+            "id": int,
+            "timestamp": str,
+            "ts_date": str,
+            "Player_id": int,
+            "total": int,
+            "attack": int,
+            "defence": int,
+            "strength": int,
+            "hitpoints": int,
+            "ranged": int,
+            "prayer": int,
+            "magic": int,
+            "cooking": int,
+            "woodcutting": int,
+            "fletching": int,
+            "fishing": int,
+            "firemaking": int,
+            "crafting": int,
+            "smithing": int,
+            "mining": int,
+            "herblore": int,
+            "agility": int,
+            "thieving": int,
+            "slayer": int,
+            "farming": int,
+            "runecraft": int,
+            "hunter": int,
+            "construction": int,
+            "league": int,
+            "bounty_hunter_hunter": int,
+            "bounty_hunter_rogue": int,
+            "cs_all": int,
+            "cs_beginner": int,
+            "cs_easy": int,
+            "cs_medium": int,
+            "cs_hard": int,
+            "cs_elite": int,
+            "cs_master": int,
+            "lms_rank": int,
+            "soul_wars_zeal": int,
+            "abyssal_sire": int,
+            "alchemical_hydra": int,
+            "barrows_chests": int,
+            "bryophyta": int,
+            "callisto": int,
+            "cerberus": int,
+            "chambers_of_xeric": int,
+            "chambers_of_xeric_challenge_mode": int,
+            "chaos_elemental": int,
+            "chaos_fanatic": int,
+            "commander_zilyana": int,
+            "corporeal_beast": int,
+            "crazy_archaeologist": int,
+            "dagannoth_prime": int,
+            "dagannoth_rex": int,
+            "dagannoth_supreme": int,
+            "deranged_archaeologist": int,
+            "general_graardor": int,
+            "giant_mole": int,
+            "grotesque_guardians": int,
+            "hespori": int,
+            "kalphite_queen": int,
+            "king_black_dragon": int,
+            "kraken": int,
+            "kreearra": int,
+            "kril_tsutsaroth": int,
+            "mimic": int,
+            "nex": int,
+            "nightmare": int,
+            "phosanis_nightmare": int,
+            "obor": int,
+            "phantom_muspah": int,
+            "sarachnis": int,
+            "scorpia": int,
+            "skotizo": int,
+            "tempoross": int,
+            "the_gauntlet": int,
+            "the_corrupted_gauntlet": int,
+            "theatre_of_blood": int,
+            "theatre_of_blood_hard": int,
+            "thermonuclear_smoke_devil": int,
+            "tombs_of_amascut": int,
+            "tombs_of_amascut_expert": int,
+            "tzkal_zuk": int,
+            "tztok_jad": int,
+            "venenatis": int,
+            "vetion": int,
+            "vorkath": int,
+            "wintertodt": int,
+            "zalcano": int,
+            "zulrah": int,
+            "rifts_closed": int,
+            "artio": int,
+            "calvarion": int,
+            "duke_sucellus": int,
+            "spindel": int,
+            "the_leviathan": int,
+            "the_whisperer": int,
+            "vardorvis": int,
+        }
+
+        # Check if the type of each value in the returned player dictionary matches the expected type
+        for key, expected_type in expected_types.items():
+            value = player.get(key)
+            # if value is not None:
+            assert isinstance(
+                value, expected_type
+            ), f"Key {key} has incorrect type. Expected: {expected_type}, Got: {type(value)}"