diff --git a/commands/owner/commands.py b/commands/owner/commands.py index a63542e8..a7183bbf 100644 --- a/commands/owner/commands.py +++ b/commands/owner/commands.py @@ -53,95 +53,8 @@ from aiohttp import TCPConnector, ClientTimeout, ClientSession import ujson from typing import Optional, List, Union, Tuple -import msgspec -import orjson -import cysimdjson - - -class Achievement(Struct, frozen=True): - name: str - stars: int - value: int - target: int - info: str - completionInfo: Union[str, None] - village: str - -class BadgeUrls(Struct, frozen=True): - small: str - medium: str - large: str - -class Clan(Struct, frozen=True): - tag: str - name: str - clanLevel: int - badgeUrls: BadgeUrls - -class Troop(Struct, frozen=True): - name: str - level: int - maxLevel: int - village: str - - -class Player(Struct, frozen=True, dict=True): - tag: str - name: str - townHallLevel: int - expLevel: int - trophies: int - bestTrophies: int - warStars: int - attackWins: int - defenseWins: int - builderBaseTrophies: int - bestBuilderBaseTrophies: int - donations: int - donationsReceived: int - clanCapitalContributions: int - achievements: List[Achievement] - heroes: List[Troop] - spells: List[Troop] - troops: List[Troop] - role: Optional[str] = None - warPreference: Optional[str] = None - clan: Optional[Clan] = None - client: coc.Client = None - - @property - def donation_total(self): - return self.donations + self.donationsReceived - -class CocPlayer(): - def __init__(self, player_struct: Player, client: coc.Client): - self.tag: str = player_struct.tag - self.name: str = player_struct.name - self.townHallLevel: int = player_struct.townHallLevel - self.expLevel: int = player_struct.expLevel - self.trophies: int = player_struct.trophies - self.bestTrophies: int = player_struct.bestTrophies - self.warStars: int = player_struct.warStars - self.attackWins: int = player_struct.attackWins - self.defenseWins: int = player_struct.defenseWins - self.builderBaseTrophies: int = player_struct.builderBaseTrophies - self.bestBuilderBaseTrophies: int = player_struct.bestBuilderBaseTrophies - self.donations: int = player_struct.donations - self.donationsReceived: int = player_struct.donationsReceived - self.clanCapitalContributions: int = player_struct.clanCapitalContributions - self.achievements: List[Achievement] = player_struct.achievements - self.heroes: List[Troop] = player_struct.troops - self.spells: List[Troop] = player_struct.troops - self.troops: List[Troop] = player_struct.troops - self.role: Optional[str] = player_struct.role - self.warPreference: Optional[str] = player_struct.warPreference - self.clan: Optional[Clan] = player_struct.clan - self.client: coc.Client = client - - @property - def donation_total(self): - return self.donations + self.donationsReceived +from testing.migrations import migrate_clan_db_simple_schema class OwnerCommands(commands.Cog): @@ -152,11 +65,6 @@ def __init__(self, bot: CustomClient): coc_client: coc.EventsClient = self.bot.coc_client - @commands.slash_command(name="restart-customs", guild_ids=[1103679645439754335]) - @commands.is_owner() - async def restart_custom(self, ctx: disnake.ApplicationCommandInteraction, top: int): - for x in range(4, top+1): - os.system(f"pm2 restart {x}") @commands.slash_command(name="exec") @@ -282,21 +190,7 @@ async def reload(self, ctx: disnake.ApplicationCommandInteraction, cog: str): @commands.slash_command(name="test", guild_ids=[1103679645439754335]) @commands.is_owner() async def test(self, ctx: disnake.ApplicationCommandInteraction): - updates = [] - print("starting") - all_documents = await self.bot.clan_stats.find({}, projection={"tag" : 1, "2024-01" : 1}).to_list(length=None) - print("got docs") - for doc in all_documents: - set_dict = {} - for tag, data in doc.get("2024-01", {}).items(): - if data.get("clan_games") is None: - continue - set_dict[f"2024-01.{tag}.clan_games"] = None - if set_dict: - updates.append(UpdateOne({"tag" : doc.get("tag")}, {"$set" : set_dict})) - print(f"{len(updates)} updates") - await self.bot.clan_stats.bulk_write(updates, ordered=False) - print("done") + await migrate_clan_db_simple_schema(bot=self.bot) diff --git a/example.env b/example.env index e69de29b..1c0640cd 100644 --- a/example.env +++ b/example.env @@ -0,0 +1,27 @@ +COC_EMAIL = email +COC_PASSWORD = password + +#mongo logins +STATIC_MONGODB = mongo_db_connection_string_#1 +STATS_MONGODB = mongo_db_connection_string_#2 + +#link api +LINK_API_USER = discord_links_user +LINK_API_PW = dicord_links_pw + +BOT_TOKEN = bot_token + +SENTRY_DSN = sentry_stats_token + +REDIS_IP = 0.0.0.0 +REDIS_PW = redis_db_pw + +BUNNY_ACCESS_KEY = bunny.net_access_token + +PORTAINER_API_TOKEN = portainer_api_token + +REDDIT_SECRET = reddit_user_secret +REDDIT_PW = reddit_user_password + +OPENAI_API_KEY = open.ai_api_token + diff --git a/testing/migrations.py b/testing/migrations.py new file mode 100644 index 00000000..2297dfdb --- /dev/null +++ b/testing/migrations.py @@ -0,0 +1,52 @@ +from classes.bot import CustomClient +from pymongo import InsertOne + + +async def migrate_clan_db_simple_schema(bot: CustomClient): + our_clan_tags = await bot.clan_db.distinct("tag") + our_clan_stats = await bot.clan_stats.find({"tag" : {"$in" : our_clan_tags}}, projection={"_id": 0}).to_list(length=None) + print(len(our_clan_stats), "clans") + + player_tags = [] + for clan_stats in our_clan_stats: #type: dict + _ = clan_stats.pop("tag") + for season, player_info in clan_stats.items(): + player_tags += list(player_info.keys()) + + player_tags = list(set(player_tags)) + print(len(player_tags), "tags") + players = await bot.get_players(tags=player_tags, custom=False) + players_map = {p.tag : p for p in players} + + bulk_insert = [] + for clan_stats in our_clan_stats: #type: dict + clan_tag = clan_stats.pop("tag") + for season, player_info in clan_stats.items(): + season_data = {"tag": clan_tag, "season" : season, "members" : []} + member_data = [] + for tag, data in player_info.items(): + api_player = players_map.get(tag) + if api_player is None: + continue + member_data.append( + { + "name" : api_player.name, + "tag" : api_player.tag, + "townhall" : api_player.town_hall, + "donated" : data.get("donated", 0), + "received": data.get("received", 0), + "activity" : data.get("activity", 0), + "gold_looted" : data.get("gold_looted", 0), + "elixir_looted" : data.get("elixir_looted", 0), + "dark_elixir_looted" : data.get("dark_elixir_looted", 0), + "attack_wins" : data.get("attack_wins", 0), + "clan_games" : data.get("clan_games", 0), + "trophies" : data.get("trophies", 0) + } + ) + season_data["members"] = member_data + bulk_insert.append(InsertOne(season_data)) + + print(len(bulk_insert), "documents") + await bot.new_looper.get_collection("new_clan_stats").bulk_write(bulk_insert) + print("done") diff --git a/tracking/example.env b/tracking/example.env new file mode 100644 index 00000000..804eed65 --- /dev/null +++ b/tracking/example.env @@ -0,0 +1,10 @@ +STATS_MONGODB = connection_string + +STATIC_MONGODB = connection_string + +COC_PASSWORD = pw + +REDIS_PW = pw + +REDIS_IP = 0.0.0.0 + diff --git a/tracking/player/constants.py b/tracking/player/constants.py new file mode 100644 index 00000000..24a165d2 --- /dev/null +++ b/tracking/player/constants.py @@ -0,0 +1,3 @@ + +SECONDARY_LOOP_CHANGE = 15 +TERTIARY_LOOP_CHANGE = 150 \ No newline at end of file diff --git a/tracking/testsocket.py b/tracking/player/tracking.py similarity index 91% rename from tracking/testsocket.py rename to tracking/player/tracking.py index 9bcf45e3..b1f04840 100644 --- a/tracking/testsocket.py +++ b/tracking/player/tracking.py @@ -1,4 +1,3 @@ -import os from typing import List, Dict, Tuple, Union from pydantic import BaseModel from base64 import b64decode as base64_b64decode @@ -9,6 +8,8 @@ from msgspec import Struct from pymongo import UpdateOne, InsertOne from datetime import timedelta + +from os import getenv from aiomultiprocess import Worker from expiring_dict import ExpiringDict from redis.commands.json.path import Path @@ -26,120 +27,33 @@ import pytz import redis from redis import asyncio as redis +import snappy + +from ..utils import create_keys -keys = [] utc = pytz.utc load_dotenv() -emails = [] -passwords = [] BETA = False -for x in range(1,13): - emails.append(f"apiclashofclans+test{x}@gmail.com") - passwords.append(os.getenv("COC_PASSWORD")) - - - -async def get_keys(emails: list, passwords: list, key_names: str, key_count: int): - total_keys = [] - - for count, email in enumerate(emails): - _keys = [] - password = passwords[count] - - session = aiohttp.ClientSession() - - body = {"email": email, "password": password} - resp = await session.post("https://developer.clashofclans.com/api/login", json=body) - if resp.status == 403: - raise RuntimeError( - "Invalid Credentials" - ) - - resp_paylaod = await resp.json() - ip = json_loads(base64_b64decode(resp_paylaod["temporaryAPIToken"].split(".")[1] + "====").decode("utf-8"))[ - "limits"][1]["cidrs"][0].split("/")[0] - - resp = await session.post("https://developer.clashofclans.com/api/apikey/list") - keys = (await resp.json())["keys"] - _keys.extend(key["key"] for key in keys if key["name"] == key_names and ip in key["cidrRanges"]) - - for key in (k for k in keys if ip not in k["cidrRanges"]): - await session.post("https://developer.clashofclans.com/api/apikey/revoke", json={"id": key["id"]}) - - print(len(_keys)) - while len(_keys) < key_count: - data = { - "name": key_names, - "description": "Created on {}".format(datetime.now().strftime("%c")), - "cidrRanges": [ip], - "scopes": ["clash"], - } - resp = await session.post("https://developer.clashofclans.com/api/apikey/create", json=data) - key = await resp.json() - _keys.append(key["key"]["key"]) - - if len(keys) == 10 and len(_keys) < key_count: - print("%s keys were requested to be used, but a maximum of %s could be " - "found/made on the developer site, as it has a maximum of 10 keys per account. " - "Please delete some keys or lower your `key_count` level." - "I will use %s keys for the life of this client.", ) - - if len(_keys) == 0: - raise RuntimeError( - "There are {} API keys already created and none match a key_name of '{}'." - "Please specify a key_name kwarg, or go to 'https://developer.clashofclans.com' to delete " - "unused keys.".format(len(keys), key_names) - ) - - await session.close() - #print("Successfully initialised keys for use.") - for k in _keys: - total_keys.append(k) - - print(len(total_keys)) - return (total_keys) - -def create_keys(): - done = False - while done is False: - try: - loop = asyncio.get_event_loop() - keys = loop.run_until_complete(get_keys(emails=emails, - passwords=passwords, key_names="test", key_count=10)) - done = True - return keys - except Exception as e: - done = False - print(e) - -class UpdateList(BaseModel): - add : List[str] - remove : List[str] - token : str - -class User(BaseModel): - username: str - password: str +keys = create_keys([f"apiclashofclans+test{x}@gmail.com" for x in range(1, 13)], [getenv("COC_PASSWORD")] * 12) async def main(producer: KafkaProducer): global keys - client = motor.motor_asyncio.AsyncIOMotorClient(os.getenv("LOOPER_DB_LOGIN")) + client = motor.motor_asyncio.AsyncIOMotorClient(getenv("LOOPER_DB_LOGIN")) player_stats = client.new_looper.player_stats clan_stats = client.new_looper.clan_stats - db_client = motor.motor_asyncio.AsyncIOMotorClient(os.getenv("DB_LOGIN")) + db_client = motor.motor_asyncio.AsyncIOMotorClient(getenv("DB_LOGIN")) clan_db = db_client.usafam.clans player_search = db_client.usafam.player_search - cache = redis.Redis(host='localhost', port=6379, db=0, password=os.getenv("REDIS_PW"), decode_responses=False, max_connections=2500) - + cache = redis.Redis(host='localhost', port=6379, db=0, password=getenv("REDIS_PW"), decode_responses=False, max_connections=2500) def get_changes(previous_response: dict, response: dict): @@ -293,12 +207,14 @@ async def fetch(url, session: aiohttp.ClientSession, headers): return None new_response = await new_response.read() + compressed_new_response = snappy.compress(new_response) + obj = decode(new_response, type=Player) previous_response = await cache.get(obj.tag) players_tracked.add(obj.tag) - if new_response != previous_response: - await cache.set(obj.tag, new_response, ex=2_592_000) + if compressed_new_response != previous_response: + await cache.set(obj.tag, compressed_new_response, ex=2_592_000) if previous_response is None: return None BEEN_ONLINE = False diff --git a/tracking/player/utils.py b/tracking/player/utils.py new file mode 100644 index 00000000..e69de29b