ClashKing v4

commands/owner/commands.py

@@ -53,95 +53,8 @@
 from aiohttp import TCPConnector, ClientTimeout, ClientSession
 import ujson
 from typing import Optional, List, Union, Tuple
-import msgspec
-import orjson
-import cysimdjson
-
-
-class Achievement(Struct, frozen=True):
-    name: str
-    stars: int
-    value: int
-    target: int
-    info: str
-    completionInfo: Union[str, None]
-    village: str
-
-class BadgeUrls(Struct, frozen=True):
-    small: str
-    medium: str
-    large: str
-
-class Clan(Struct, frozen=True):
-    tag: str
-    name: str
-    clanLevel: int
-    badgeUrls: BadgeUrls
-
-class Troop(Struct, frozen=True):
-    name: str
-    level: int
-    maxLevel: int
-    village: str
-
-
-class Player(Struct, frozen=True, dict=True):
-    tag: str
-    name: str
-    townHallLevel: int
-    expLevel: int
-    trophies: int
-    bestTrophies: int
-    warStars: int
-    attackWins: int
-    defenseWins: int
-    builderBaseTrophies: int
-    bestBuilderBaseTrophies: int
-    donations: int
-    donationsReceived: int
-    clanCapitalContributions: int
-    achievements: List[Achievement]
-    heroes: List[Troop]
-    spells: List[Troop]
-    troops: List[Troop]
-    role: Optional[str] = None
-    warPreference: Optional[str] = None
-    clan: Optional[Clan] = None
-    client: coc.Client = None
-
-    @property
-    def donation_total(self):
-        return self.donations + self.donationsReceived
-
-class CocPlayer():
-    def __init__(self, player_struct: Player, client: coc.Client):
-        self.tag: str = player_struct.tag
-        self.name: str = player_struct.name
-        self.townHallLevel: int = player_struct.townHallLevel
-        self.expLevel: int = player_struct.expLevel
-        self.trophies: int = player_struct.trophies
-        self.bestTrophies: int = player_struct.bestTrophies
-        self.warStars: int = player_struct.warStars
-        self.attackWins: int = player_struct.attackWins
-        self.defenseWins: int = player_struct.defenseWins
-        self.builderBaseTrophies: int = player_struct.builderBaseTrophies
-        self.bestBuilderBaseTrophies: int = player_struct.bestBuilderBaseTrophies
-        self.donations: int = player_struct.donations
-        self.donationsReceived: int = player_struct.donationsReceived
-        self.clanCapitalContributions: int = player_struct.clanCapitalContributions
-        self.achievements: List[Achievement] = player_struct.achievements
-        self.heroes: List[Troop] = player_struct.troops
-        self.spells: List[Troop] = player_struct.troops
-        self.troops: List[Troop] = player_struct.troops
-        self.role: Optional[str] = player_struct.role
-        self.warPreference: Optional[str] = player_struct.warPreference
-        self.clan: Optional[Clan] = player_struct.clan
-        self.client: coc.Client = client
-
-    @property
-    def donation_total(self):
-        return self.donations + self.donationsReceived
 
+from testing.migrations import migrate_clan_db_simple_schema
 
 
 class OwnerCommands(commands.Cog):
@@ -152,11 +65,6 @@ def __init__(self, bot: CustomClient):
         coc_client: coc.EventsClient = self.bot.coc_client
 
 
-    @commands.slash_command(name="restart-customs", guild_ids=[1103679645439754335])
-    @commands.is_owner()
-    async def restart_custom(self, ctx: disnake.ApplicationCommandInteraction, top: int):
-        for x in range(4, top+1):
-            os.system(f"pm2 restart {x}")
 
 
     @commands.slash_command(name="exec")
@@ -282,21 +190,7 @@ async def reload(self, ctx: disnake.ApplicationCommandInteraction, cog: str):
     @commands.slash_command(name="test", guild_ids=[1103679645439754335])
     @commands.is_owner()
     async def test(self, ctx: disnake.ApplicationCommandInteraction):
-        updates = []
-        print("starting")
-        all_documents = await self.bot.clan_stats.find({}, projection={"tag" : 1, "2024-01" : 1}).to_list(length=None)
-        print("got docs")
-        for doc in all_documents:
-            set_dict = {}
-            for tag, data in doc.get("2024-01", {}).items():
-                if data.get("clan_games") is None:
-                    continue
-                set_dict[f"2024-01.{tag}.clan_games"] = None
-            if set_dict:
-                updates.append(UpdateOne({"tag" : doc.get("tag")}, {"$set" : set_dict}))
-        print(f"{len(updates)} updates")
-        await self.bot.clan_stats.bulk_write(updates, ordered=False)
-        print("done")
+        await migrate_clan_db_simple_schema(bot=self.bot)
 
 
 

example.env

@@ -0,0 +1,27 @@
+COC_EMAIL = email
+COC_PASSWORD = password
+
+#mongo logins
+STATIC_MONGODB = mongo_db_connection_string_#1
+STATS_MONGODB = mongo_db_connection_string_#2
+
+#link api
+LINK_API_USER = discord_links_user
+LINK_API_PW = dicord_links_pw
+
+BOT_TOKEN = bot_token
+
+SENTRY_DSN = sentry_stats_token
+
+REDIS_IP = 0.0.0.0
+REDIS_PW = redis_db_pw
+
+BUNNY_ACCESS_KEY = bunny.net_access_token
+
+PORTAINER_API_TOKEN = portainer_api_token
+
+REDDIT_SECRET = reddit_user_secret
+REDDIT_PW = reddit_user_password
+
+OPENAI_API_KEY = open.ai_api_token
+

testing/migrations.py

@@ -0,0 +1,52 @@
+from classes.bot import CustomClient
+from pymongo import InsertOne
+
+
+async def migrate_clan_db_simple_schema(bot: CustomClient):
+    our_clan_tags = await bot.clan_db.distinct("tag")
+    our_clan_stats = await bot.clan_stats.find({"tag" : {"$in" : our_clan_tags}}, projection={"_id": 0}).to_list(length=None)
+    print(len(our_clan_stats), "clans")
+
+    player_tags = []
+    for clan_stats in our_clan_stats: #type: dict
+        _ = clan_stats.pop("tag")
+        for season, player_info in clan_stats.items():
+            player_tags += list(player_info.keys())
+
+    player_tags = list(set(player_tags))
+    print(len(player_tags), "tags")
+    players = await bot.get_players(tags=player_tags, custom=False)
+    players_map = {p.tag : p for p in players}
+
+    bulk_insert = []
+    for clan_stats in our_clan_stats: #type: dict
+        clan_tag = clan_stats.pop("tag")
+        for season, player_info in clan_stats.items():
+            season_data = {"tag": clan_tag, "season" : season, "members" : []}
+            member_data = []
+            for tag, data in player_info.items():
+                api_player = players_map.get(tag)
+                if api_player is None:
+                    continue
+                member_data.append(
+                    {
+                        "name" : api_player.name,
+                        "tag" : api_player.tag,
+                        "townhall" : api_player.town_hall,
+                        "donated" : data.get("donated", 0),
+                        "received": data.get("received", 0),
+                        "activity" : data.get("activity", 0),
+                        "gold_looted" : data.get("gold_looted", 0),
+                        "elixir_looted" : data.get("elixir_looted", 0),
+                        "dark_elixir_looted" : data.get("dark_elixir_looted", 0),
+                        "attack_wins" : data.get("attack_wins", 0),
+                        "clan_games" : data.get("clan_games", 0),
+                        "trophies" : data.get("trophies", 0)
+                    }
+                )
+            season_data["members"] = member_data
+            bulk_insert.append(InsertOne(season_data))
+
+    print(len(bulk_insert), "documents")
+    await bot.new_looper.get_collection("new_clan_stats").bulk_write(bulk_insert)
+    print("done")

tracking/example.env

@@ -0,0 +1,10 @@
+STATS_MONGODB = connection_string
+
+STATIC_MONGODB = connection_string
+
+COC_PASSWORD = pw
+
+REDIS_PW = pw
+
+REDIS_IP = 0.0.0.0
+

tracking/player/constants.py

@@ -0,0 +1,3 @@
+
+SECONDARY_LOOP_CHANGE = 15
+TERTIARY_LOOP_CHANGE = 150

tracking/testsocket.py → tracking/player/tracking.py

@@ -1,4 +1,3 @@
-import os
 from typing import List, Dict, Tuple, Union
 from pydantic import BaseModel
 from base64 import b64decode as base64_b64decode
@@ -9,6 +8,8 @@
 from msgspec import Struct
 from pymongo import UpdateOne, InsertOne
 from datetime import timedelta
+
+from os import getenv
 from aiomultiprocess import Worker
 from expiring_dict import ExpiringDict
 from redis.commands.json.path import Path
@@ -26,120 +27,33 @@
 import pytz
 import redis
 from redis import asyncio as redis
+import snappy
+
+from ..utils import create_keys
 
-keys = []
 utc = pytz.utc
 load_dotenv()
-emails = []
-passwords = []
 
 BETA = False
 
 
-for x in range(1,13):
-    emails.append(f"apiclashofclans+test{x}@gmail.com")
-    passwords.append(os.getenv("COC_PASSWORD"))
-
-
-
-async def get_keys(emails: list, passwords: list, key_names: str, key_count: int):
-    total_keys = []
-
-    for count, email in enumerate(emails):
-        _keys = []
-        password = passwords[count]
-
-        session = aiohttp.ClientSession()
-
-        body = {"email": email, "password": password}
-        resp = await session.post("https://developer.clashofclans.com/api/login", json=body)
-        if resp.status == 403:
-            raise RuntimeError(
-                "Invalid Credentials"
-            )
-
-        resp_paylaod = await resp.json()
-        ip = json_loads(base64_b64decode(resp_paylaod["temporaryAPIToken"].split(".")[1] + "====").decode("utf-8"))[
-            "limits"][1]["cidrs"][0].split("/")[0]
-
-        resp = await session.post("https://developer.clashofclans.com/api/apikey/list")
-        keys = (await resp.json())["keys"]
-        _keys.extend(key["key"] for key in keys if key["name"] == key_names and ip in key["cidrRanges"])
-
-        for key in (k for k in keys if ip not in k["cidrRanges"]):
-            await session.post("https://developer.clashofclans.com/api/apikey/revoke", json={"id": key["id"]})
-
-        print(len(_keys))
-        while len(_keys) < key_count:
-            data = {
-                "name": key_names,
-                "description": "Created on {}".format(datetime.now().strftime("%c")),
-                "cidrRanges": [ip],
-                "scopes": ["clash"],
-            }
-            resp = await session.post("https://developer.clashofclans.com/api/apikey/create", json=data)
-            key = await resp.json()
-            _keys.append(key["key"]["key"])
-
-        if len(keys) == 10 and len(_keys) < key_count:
-            print("%s keys were requested to be used, but a maximum of %s could be "
-                  "found/made on the developer site, as it has a maximum of 10 keys per account. "
-                  "Please delete some keys or lower your `key_count` level."
-                  "I will use %s keys for the life of this client.", )
-
-        if len(_keys) == 0:
-            raise RuntimeError(
-                "There are {} API keys already created and none match a key_name of '{}'."
-                "Please specify a key_name kwarg, or go to 'https://developer.clashofclans.com' to delete "
-                "unused keys.".format(len(keys), key_names)
-            )
-
-        await session.close()
-        #print("Successfully initialised keys for use.")
-        for k in _keys:
-            total_keys.append(k)
-
-    print(len(total_keys))
-    return (total_keys)
-
-def create_keys():
-    done = False
-    while done is False:
-        try:
-            loop = asyncio.get_event_loop()
-            keys = loop.run_until_complete(get_keys(emails=emails,
-                                     passwords=passwords, key_names="test", key_count=10))
-            done = True
-            return keys
-        except Exception as e:
-            done = False
-            print(e)
-
-class UpdateList(BaseModel):
-    add : List[str]
-    remove : List[str]
-    token : str
-
-class User(BaseModel):
-    username: str
-    password: str
+keys = create_keys([f"apiclashofclans+test{x}@gmail.com" for x in range(1, 13)], [getenv("COC_PASSWORD")] * 12)
 
 
 
 async def main(producer: KafkaProducer):
     global keys
 
-    client = motor.motor_asyncio.AsyncIOMotorClient(os.getenv("LOOPER_DB_LOGIN"))
+    client = motor.motor_asyncio.AsyncIOMotorClient(getenv("LOOPER_DB_LOGIN"))
     player_stats = client.new_looper.player_stats
     clan_stats = client.new_looper.clan_stats
 
-    db_client = motor.motor_asyncio.AsyncIOMotorClient(os.getenv("DB_LOGIN"))
+    db_client = motor.motor_asyncio.AsyncIOMotorClient(getenv("DB_LOGIN"))
     clan_db = db_client.usafam.clans
     player_search = db_client.usafam.player_search
 
 
-    cache = redis.Redis(host='localhost', port=6379, db=0, password=os.getenv("REDIS_PW"), decode_responses=False, max_connections=2500)
-
+    cache = redis.Redis(host='localhost', port=6379, db=0, password=getenv("REDIS_PW"), decode_responses=False, max_connections=2500)
 
 
     def get_changes(previous_response: dict, response: dict):
@@ -293,12 +207,14 @@ async def fetch(url, session: aiohttp.ClientSession, headers):
                                 return None
                             new_response = await new_response.read()
 
+                            compressed_new_response = snappy.compress(new_response)
+
                             obj = decode(new_response, type=Player)
                             previous_response = await cache.get(obj.tag)
                             players_tracked.add(obj.tag)
 
-                            if new_response != previous_response:
-                                await cache.set(obj.tag, new_response, ex=2_592_000)
+                            if compressed_new_response != previous_response:
+                                await cache.set(obj.tag, compressed_new_response, ex=2_592_000)
                                 if previous_response is None:
                                     return None
                                 BEEN_ONLINE = False