simplify setup

api/cogs/requests.py

@@ -105,6 +105,25 @@ async def get_player_data(label_id: int, limit: int = 5000):
     return players
 
 
+def parse_hs_data(data: list[ScraperDataV3]) -> list[HighscoreData]:
+    hiscores = []
+    for d in data:
+        scraper_data = ScraperDataV3(**d)
+        skills = {r.skill_name: r.skill_value for r in scraper_data.skills}
+        activities = {
+            r.activity_name: r.activity_value for r in scraper_data.activities
+        }
+        hiscores.append(
+            HighscoreData(
+                **skills,
+                **activities,
+                Player_id=scraper_data.player_id,
+                name=scraper_data.player_name,
+            )
+        )
+    return hiscores
+
+
 async def get_hiscore_data(label_id: int, limit: int = 5000):
     url = f"{config.private_api}/v3/highscore/latest"
     params = {"player_id": 1, "label_id": label_id, "many": 1, "limit": limit}
@@ -115,21 +134,10 @@ async def get_hiscore_data(label_id: int, limit: int = 5000):
     # Continue making requests until all data is retrieved
     while True:
         data = await retry_request(url=url, params=params)
-
-        for d in data:
-            scraper_data = ScraperDataV3(**d)
-            skills = {r.skill_name: r.skill_value for r in scraper_data.skills}
-            activities = {
-                r.activity_name: r.activity_value for r in scraper_data.activities
-            }
-            hiscores.append(
-                HighscoreData(
-                    **skills,
-                    **activities,
-                    Player_id=scraper_data.player_id,
-                    name=scraper_data.player_name,
-                ).model_dump()
-            )
+        data = [ScraperDataV3(**d) for d in data]
+        _hiscores = parse_hs_data(data=data)
+        _hiscores = [h.model_dump() for h in _hiscores]
+        hiscores.extend(_hiscores)
 
         logger.info(f"received: {len(data)}, in total {len(hiscores)}")
 
@@ -139,16 +147,7 @@ async def get_hiscore_data(label_id: int, limit: int = 5000):
 
         # Increment the page parameter for the next request
         last_record = data[-1]
-        if not isinstance(last_record, dict):
-            logger.error(f"expected dict but got {type(last_record)}, {last_record=}")
-            break
-
-        last_player_id = last_record.get("Player_id", None)
-        if last_player_id is None:
-            logger.error(f"expected int but got None, {last_record=}")
-            break
-
-        params["player_id"] = last_player_id
+        params["player_id"] = last_record.player_id
 
     return hiscores
 
@@ -158,21 +157,9 @@ async def get_prediction_data(player_id: int = 0, limit: int = 0):
     params = {"player_id": player_id, "many": 1, "limit": limit}
 
     data = await retry_request(url=url, params=params)
-    hiscores = []
-    for d in data:
-        scraper_data = ScraperDataV3(**d)
-        skills = {r.skill_name: r.skill_value for r in scraper_data.skills}
-        activities = {
-            r.activity_name: r.activity_value for r in scraper_data.activities
-        }
-        hiscores.append(
-            HighscoreData(
-                **skills,
-                **activities,
-                Player_id=scraper_data.player_id,
-                name=scraper_data.player_name,
-            ).model_dump()
-        )
+    data = [ScraperDataV3(**d) for d in data]
+    hiscores = parse_hs_data(data=data)
+    hiscores = [h.model_dump() for h in hiscores]
     return hiscores