diff --git a/cogs/assistance.py b/cogs/assistance.py index ce900628c..fa9da2952 100644 --- a/cogs/assistance.py +++ b/cogs/assistance.py @@ -1,6 +1,6 @@ -import aiohttp import discord import qrcode +import logging from discord.ext import commands, tasks from io import BytesIO @@ -10,6 +10,9 @@ from utils.checks import check_if_user_can_sr +logger = logging.getLogger(__name__) + + class Assistance(commands.Cog, command_attrs=dict(cooldown=commands.Cooldown(1, 30.0, commands.BucketType.channel))): """ Commands that will mostly be used in the help channels. @@ -28,14 +31,14 @@ def __init__(self, bot): @tasks.loop(hours=2) async def apps_update(self): - async with aiohttp.ClientSession() as session: - r = await session.get('https://raw.githubusercontent.com/Universal-Team/db/master/docs/data/full.json') + async with self.bot.session.get("https://raw.githubusercontent.com/Universal-Team/db/master/docs/data/full.json", timeout=45) as r: if r.status == 200: # Content type is text/plain instead of application/json self.unidb = await r.json(content_type=None) + logger.info("Downloaded Universal Team Database") else: self.unidb = {} - print("Failed to fetch unidb.") + logger.warning("Failed to fetch Universal Team Database.") def unisearch(self, query: str) -> dict: query = query.lower() diff --git a/cogs/imgconvert.py b/cogs/imgconvert.py index 1dada36bc..43006fe2f 100644 --- a/cogs/imgconvert.py +++ b/cogs/imgconvert.py @@ -1,4 +1,3 @@ -import aiohttp import concurrent.futures import functools @@ -28,15 +27,14 @@ async def on_message(self, message): # BMP conversion for f in message.attachments: if f.filename.lower().endswith('.bmp') and f.size <= 600000: # 600kb - async with aiohttp.ClientSession() as session: - async with session.get(f.url, timeout=45) as img_request: - img_content = await img_request.read() - with concurrent.futures.ProcessPoolExecutor() as pool: - img_out = await self.bot.loop.run_in_executor(pool, functools.partial(self.img_convert, img_content)) - out_message = f"{f.filename} from {message.author.mention}" - new_filename = f.filename[:-3] + "png" - img = File(img_out, filename=new_filename) - await message.channel.send(file=img, content=out_message) + async with self.bot.session.get(f.url, timeout=45) as img_request: + img_content = await img_request.read() + with concurrent.futures.ProcessPoolExecutor() as pool: + img_out = await self.bot.loop.run_in_executor(pool, functools.partial(self.img_convert, img_content)) + out_message = f"{f.filename} from {message.author.mention}" + new_filename = f.filename[:-3] + "png" + img = File(img_out, filename=new_filename) + await message.channel.send(file=img, content=out_message) def setup(bot): diff --git a/cogs/loop.py b/cogs/loop.py index cbc199df8..19309b3f3 100644 --- a/cogs/loop.py +++ b/cogs/loop.py @@ -1,9 +1,7 @@ -import aiohttp import asyncio import discord import pytz -import sys -import traceback +import logging from datetime import datetime, timedelta from discord.ext import commands @@ -11,6 +9,9 @@ from utils.utils import send_dm_message +logger = logging.getLogger(__name__) + + class Loop(commands.Cog): """ Loop events. @@ -42,13 +43,12 @@ def netinfo_parse_time(self, timestr): return datetime.strptime(' '.join(timestr.split()), '%A, %B %d, %Y %I :%M %p').replace(tzinfo=self.tz) async def update_netinfo(self): - async with aiohttp.ClientSession() as session: - r = await session.get('https://www.nintendo.co.jp/netinfo/en_US/status.json?callback=getJSON') + async with self.bot.session.get('https://www.nintendo.co.jp/netinfo/en_US/status.json?callback=getJSON', timeout=45) as r: if r.status == 200: j = await r.json() else: - # No logging setup :/ - print(f"Netinfo: {r.status} while trying to update netinfo.") + # logging setup :) + logger.warning("Status %s while trying to update netinfo.", r.status) return now = datetime.now(self.tz) @@ -228,10 +228,8 @@ async def start_update_loop(self): if current_timestamp.minute % 30 == 0 and current_timestamp.second == 0: self.bot.loop.create_task(self.update_netinfo()) - except Exception as e: - print('Ignoring exception in start_update_loop', file=sys.stderr) - traceback.print_tb(e.__traceback__) - print(f'{e.__class__.__name__}: {e}', file=sys.stderr) + except Exception: + logger.error("Ignoring exception in start_update_loop", exc_info=True) finally: await asyncio.sleep(1) diff --git a/kurisu.py b/kurisu.py index 52d5be755..ff0d5f75f 100644 --- a/kurisu.py +++ b/kurisu.py @@ -4,34 +4,56 @@ # license: Apache License 2.0 # https://github.com/nh-server/Kurisu -from asyncio import Event -from alembic import config, script, command -from alembic.runtime import migration -from configparser import ConfigParser -from datetime import datetime -from discord.ext import commands -from sqlalchemy import engine -from subprocess import check_output, CalledProcessError -from sys import exit, hexversion -from traceback import format_exc - +import aiohttp +import asyncio import discord +import gino +import logging import os import sys +import traceback -from utils import models, crud +from alembic.config import main as albmain +from configparser import ConfigParser +from discord.ext import commands +from logging.handlers import TimedRotatingFileHandler +from subprocess import check_output, CalledProcessError +from utils import crud from utils.checks import check_staff_id -from utils.manager import WordFilterManager, InviteFilterManager, LevenshteinFilterManager -from utils.models import db -from utils.utils import create_error_embed, paginate_message +from utils.manager import InviteFilterManager, WordFilterManager, LevenshteinFilterManager +from utils.models import Channel, Role, db +from utils.utils import create_error_embed -IS_DOCKER = os.environ.get('IS_DOCKER', '') +cogs = ( + 'cogs.assistance', + 'cogs.blah', + 'cogs.events', + 'cogs.extras', + 'cogs.filters', + 'cogs.friendcode', + 'cogs.kickban', + 'cogs.load', + 'cogs.lockdown', + 'cogs.logs', + 'cogs.loop', + 'cogs.memes', + 'cogs.helperlist', + 'cogs.imgconvert', + 'cogs.mod_staff', + 'cogs.mod_warn', + 'cogs.mod_watch', + 'cogs.mod', + 'cogs.results', + 'cogs.rules', + 'cogs.ssnc', + 'cogs.xkcdparse', + 'cogs.seasonal', + 'cogs.newcomers', +) -# sets working directory to bot's folder -dir_path = os.path.dirname(os.path.realpath(__file__)) -os.chdir(dir_path) +DEBUG = False +IS_DOCKER = os.environ.get('IS_DOCKER', '') -# Load config if IS_DOCKER: def get_env(name: str): contents = os.environ.get(name) @@ -55,50 +77,37 @@ def get_env(name: str): TOKEN = kurisu_config['Main']['token'] DATABASE_URL = kurisu_config['Main']['database_url'] -# loads extensions -cogs = [ - 'cogs.assistance', - 'cogs.blah', - 'cogs.events', - 'cogs.extras', - 'cogs.filters', - 'cogs.friendcode', - 'cogs.kickban', - 'cogs.load', - 'cogs.lockdown', - 'cogs.logs', - 'cogs.loop', - 'cogs.memes', - 'cogs.helperlist', - 'cogs.imgconvert', - 'cogs.mod_staff', - 'cogs.mod_warn', - 'cogs.mod_watch', - 'cogs.mod', - 'cogs.results', - 'cogs.rules', - 'cogs.ssnc', - 'cogs.xkcdparse', - 'cogs.seasonal', - 'cogs.newcomers', -] +def setup_logging(): + log = logging.getLogger() + log.setLevel(logging.INFO) + fh = TimedRotatingFileHandler(filename='data/kurisu.log', when='midnight', encoding='utf-8', backupCount=7) + fmt = logging.Formatter('[{asctime}] [{levelname:^7s}] {module}.{funcName}: {message}', datefmt="%Y-%m-%d %H:%M:%S", + style='{') + fh.setFormatter(fmt) + log.addHandler(fh) + sh = logging.StreamHandler() + sh.setFormatter(fmt) + log.addHandler(sh) + logging.getLogger('discord').propagate = False + logging.getLogger('gino').propagate = False -class CustomContext(commands.Context): - async def get_user(self, userid: int): - if self.guild and (user := self.guild.get_member(userid)): - return user - else: - return await self.bot.fetch_user(userid) + +logger = logging.getLogger(__name__) class Kurisu(commands.Bot): - """Its him!!.""" - def __init__(self, *args, commit, branch, **kwargs): - super().__init__(*args, **kwargs) - self.startup = datetime.now() + def __init__(self, command_prefix, description, commit, branch): + intents = discord.Intents(guilds=True, members=True, messages=True, reactions=True) + allowed_mentions = discord.AllowedMentions(everyone=False, roles=False) + super().__init__( + command_prefix=command_prefix, + description=description, + intents=intents, + allowed_mentions=allowed_mentions + ) self.IS_DOCKER = IS_DOCKER self.commit = commit self.branch = branch @@ -136,9 +145,6 @@ def __init__(self, *args, commit, branch, **kwargs): '🍰': None, } - self.actions = [] - self.pruning = False - self.channels = { 'announcements': None, 'welcome-and-rules': None, @@ -165,139 +171,152 @@ def __init__(self, *args, commit, branch, **kwargs): 'mod-logs': None, 'server-logs': None, 'bot-err': None, - 'elsewhere': None, # I'm a bit worried about how often this changes, shouldn't be a problem tho + 'elsewhere': None, 'newcomers': None, 'nintendo-discussion': None, 'tech-talk': None, 'hardware': None, } + self.assistance_channels = ( + self.channels['3ds-assistance-1'], + self.channels['3ds-assistance-2'], + self.channels['wiiu-assistance'], + self.channels['switch-assistance-1'], + self.channels['switch-assistance-2'], + self.channels['hacking-general'], + self.channels['legacy-systems'], + self.channels['tech-talk'], + self.channels['hardware'], + ) + + self.staff_roles = {'Owner': self.roles['Owner'], + 'SuperOP': self.roles['SuperOP'], + 'OP': self.roles['OP'], + 'HalfOP': self.roles['HalfOP'], + 'Staff': self.roles['Staff'], + } + + self.helper_roles = {"3DS": self.roles['On-Duty 3DS'], + "WiiU": self.roles['On-Duty Wii U'], + "Switch": self.roles['On-Duty Switch'], + "Legacy": self.roles['On-Duty Legacy'] + } + self.failed_cogs = [] - self.exitcode = 0 - self._is_all_ready = Event() + self.channels_not_found = [] + self.roles_not_found = [] + + self.wordfilter = WordFilterManager() + self.levenshteinfilter = LevenshteinFilterManager() + self.invitefilter = InviteFilterManager() + + self.guild = None + self.err_channel = None + + self.session = aiohttp.ClientSession(loop=self.loop) + + self._is_all_ready = asyncio.Event() + self.load_cogs() + + async def on_ready(self): + + if self._is_all_ready.is_set(): + return + self.err_channel = self.channels['bot-err'] + self.guild = self.guilds[0] + + # Load Filters + await self.wordfilter.load() + logger.info("Loaded wordfilter") + await self.levenshteinfilter.load() + logger.info("Loaded levenshtein filter") + await self.invitefilter.load() + logger.info("Loaded invite filter") - os.makedirs("data", exist_ok=True) + # Load channels and roles + await self.load_channels() + await self.load_roles() - async def get_context(self, message, *, cls=CustomContext): - return await super().get_context(message, cls=cls) + startup_message = f'{self.user.name} has started! {self.guild} has {self.guild.member_count:,} members!' + embed = discord.Embed(title=f"{self.user.name} has started!", + description=f"{self.guild} has {self.guild.member_count:,} members!", colour=0xb01ec3) + if self.failed_cogs or self.roles_not_found or self.channels_not_found: + embed.colour = 0xe50730 + if self.failed_cogs: + embed.add_field( + name="Failed to load cogs:", + value='\n'.join( + f"**{cog}**\n**{exc_type}**: {exc}\n" for cog, exc_type, exc in self.failed_cogs + ), + inline=False, + ) + if self.roles_not_found: + embed.add_field(name="Roles not Found:", value=', '.join(self.roles_not_found), inline=False) + if self.channels_not_found: + embed.add_field(name="Channels not Found:", value=', '.join(self.channels_not_found), inline=False) + + logger.info(startup_message) + await self.channels['helpers'].send(embed=embed) - def upgrade_database_revision(self): - connection = engine.create_engine(DATABASE_URL) - alembic_cfg = config.Config('./alembic.ini', stdout=None) - directory = script.ScriptDirectory.from_config(alembic_cfg) - with connection.begin() as connection: - context = migration.MigrationContext.configure(connection) - if set(context.get_current_heads()) != set(directory.get_heads()): - print('Upgrading database revision') - command.upgrade(alembic_cfg, 'head') + self._is_all_ready.set() + + async def wait_until_all_ready(self): + """Wait until the bot is finished setting up.""" + await self._is_all_ready.wait() + + @staticmethod + def escape_text(text: str): + return discord.utils.escape_markdown(text) + + async def close(self): + await super().close() + await db.pop_bind().close() + await self.session.close() def load_cogs(self): for extension in cogs: try: self.load_extension(extension) except BaseException as e: - print(f'{extension} failed to load.') - self.failed_cogs.append([extension, type(e).__name__, e]) + logger.error("%s failed to load.", extension) + self.failed_cogs.append((extension, type(e).__name__, e)) async def load_channels(self): for n in self.channels: - if channel := await models.Channel.query.where(models.Channel.name == n).gino.scalar(): + if channel := await Channel.query.where(Channel.name == n).gino.scalar(): self.channels[n] = self.guild.get_channel(channel) else: self.channels[n] = discord.utils.get(self.guild.text_channels, name=n) if not self.channels[n]: - print(f"Failed to find channel {n}") + self.channels_not_found.append(n) + logger.warning("Failed to find channel %s", n) continue if db_chan := await crud.get_dbchannel(self.channels[n].id): await db_chan.update(name=n).apply() else: - await models.Channel.create(id=self.channels[n].id, name=self.channels[n].name) + await Channel.create(id=self.channels[n].id, name=self.channels[n].name) async def load_roles(self): for n in self.roles: - if role := await models.Role.query.where(models.Role.name == n).gino.scalar(): + if role := await Role.query.where(Role.name == n).gino.scalar(): self.roles[n] = self.guild.get_role(role) else: self.roles[n] = discord.utils.get(self.guild.roles, name=n) if not self.roles[n]: - print(f"Failed to find role {n}") + self.roles_not_found.append(n) + logger.warning("Failed to find role %s", n) continue if db_role := await crud.get_dbrole(self.roles[n].id): await db_role.update(name=n).apply() else: - await models.Role.create(id=self.roles[n].id, name=self.roles[n].name) - # Nitro Booster existence depends if there is any nitro booster - self.roles['Nitro Booster'] = self.guild.premium_subscriber_role - if self.roles['Nitro Booster'] and not await crud.get_dbrole(self.roles['Nitro Booster'].id): - await models.Role.create(id=self.roles['Nitro Booster'].id, name='Nitro Booster') - - @staticmethod - def escape_text(text): - text = str(text) - return discord.utils.escape_markdown(text) - - async def on_ready(self): - guilds = self.guilds - assert len(guilds) == 1 - self.guild = guilds[0] + await Role.create(id=self.roles[n].id, name=self.roles[n].name) - try: - self.upgrade_database_revision() - await db.set_bind(DATABASE_URL) - except: - sys.exit('Error when connecting to database') - - await self.load_channels() - await self.load_roles() - - self.assistance_channels = { - self.channels['3ds-assistance-1'], - self.channels['3ds-assistance-2'], - self.channels['wiiu-assistance'], - self.channels['switch-assistance-1'], - self.channels['switch-assistance-2'], - self.channels['hacking-general'], - self.channels['legacy-systems'], - self.channels['tech-talk'], - self.channels['hardware'], - } - - self.staff_roles = {'Owner': self.roles['Owner'], - 'SuperOP': self.roles['SuperOP'], - 'OP': self.roles['OP'], - 'HalfOP': self.roles['HalfOP'], - 'Staff': self.roles['Staff'], - } - - self.helper_roles = {"3DS": self.roles['On-Duty 3DS'], - "WiiU": self.roles['On-Duty Wii U'], - "Switch": self.roles['On-Duty Switch'], - "Legacy": self.roles['On-Duty Legacy'] - } - - self.wordfilter = WordFilterManager() - await self.wordfilter.load() - - self.levenshteinfilter = LevenshteinFilterManager() - await self.levenshteinfilter.load() - - self.invitefilter = InviteFilterManager() - await self.invitefilter.load() - - startup_message = f'{self.user.name} has started! {self.guild} has {self.guild.member_count:,} members!' - if len(self.failed_cogs) != 0: - startup_message += "\n\nSome addons failed to load:\n" - for f in self.failed_cogs: - startup_message += "\n{}: `{}: {}`".format(*f) - print(startup_message) - await self.channels['helpers'].send(startup_message) - self._is_all_ready.set() - - async def on_command_error(self, ctx: commands.Context, exc: commands.CommandInvokeError): + async def on_command_error(self, ctx: commands.Context, exc: discord.DiscordException): author: discord.Member = ctx.author command: commands.Command = ctx.command or '' exc = getattr(exc, 'original', exc) - channel = self.channels['bot-err'] if self.channels['bot-err'] else ctx.channel + channel = self.err_channel or ctx.channel if isinstance(exc, commands.CommandNotFound): return @@ -327,7 +346,8 @@ async def on_command_error(self, ctx: commands.Context, exc: commands.CommandInv await ctx.message.delete() except (discord.errors.NotFound, discord.errors.Forbidden): pass - await ctx.send(f"{author.mention} This command was used {exc.cooldown.per - exc.retry_after:.2f}s ago and is on cooldown. Try again in {exc.retry_after:.2f}s.", delete_after=10) + await ctx.send(f"{author.mention} This command was used {exc.cooldown.per - exc.retry_after:.2f}s ago and is on cooldown.\ + Try again in {exc.retry_after:.2f}s.", delete_after=10) else: await ctx.reinvoke() @@ -351,38 +371,27 @@ async def on_command_error(self, ctx: commands.Context, exc: commands.CommandInv await channel.send(embed=embed) async def on_error(self, event_method, *args, **kwargs): - await self.channels['bot-err'].send(f'Error in {event_method}:') - msg = format_exc() - error_paginator = paginate_message(msg) - for page in error_paginator.pages: - await self.channels['bot-err'].send(page) - - def add_cog(self, cog): - super().add_cog(cog) - print(f'Cog "{cog.qualified_name}" loaded') - - async def close(self): - print('Kurisu is shutting down') - await db.pop_bind().close() - await super().close() + logger.error("", exc_info=True) + if not self.err_channel: + return - async def is_all_ready(self): - """Checks if the bot is finished setting up.""" - return self._is_all_ready.is_set() + exc_type, exc, tb = sys.exc_info() + embed = discord.Embed(title="Error Event", colour=0xe50730) + embed.add_field(name="Event Method", value=event_method) + trace = "".join(traceback.format_exception(exc_type, exc, tb)) + embed.description = f"```py\n{trace}\n```" + await self.err_channel.send(embed=embed) - async def wait_until_all_ready(self): - """Wait until the bot is finished setting up.""" - await self._is_all_ready.wait() +async def startup(): + setup_logging() -def main(): - """Main script to run the bot.""" if discord.version_info.major < 1: - print(f'discord.py is not at least 1.0.0x. (current version: {discord.__version__})') + logger.error("discord.py is not at least 1.0.0x. (current version: %s)", discord.__version__) return 2 - if not hexversion >= 0x30800f0: # 3.8 - print('Kurisu requires 3.8 or later.') + if sys.hexversion < 0x30900F0: # 3.9 + logger.error("Kurisu requires 3.9 or later.") return 2 if not IS_DOCKER: @@ -390,28 +399,32 @@ def main(): try: commit = check_output(['git', 'rev-parse', 'HEAD']).decode('ascii')[:-1] except CalledProcessError as e: - print(f'Checking for git commit failed: {type(e).__name__}: {e}') + logger.error("Checking for git commit failed: %s: %s", type(e).__name__, e) commit = "" try: branch = check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode()[:-1] except CalledProcessError as e: - print(f'Checking for git branch failed: {type(e).__name__}: {e}') + logger.error("Checking for git branch failed: %s: %s", type(e).__name__, e) branch = "" else: commit = os.environ.get('COMMIT_SHA') branch = os.environ.get('COMMIT_BRANCH') - intents = discord.Intents(guilds=True, members=True, bans=True, messages=True) - - bot = Kurisu(('.', '!'), description="Kurisu, the bot for Nintendo Homebrew!", allowed_mentions=discord.AllowedMentions(everyone=False, roles=False), commit=commit, branch=branch, intents=intents, case_insensitive=True) - bot.help_command = commands.DefaultHelpCommand(dm_help=None) - print(f'Starting Kurisu on commit {commit} on branch {branch}') - bot.load_cogs() - bot.run(TOKEN) - - return bot.exitcode + try: + albmain(['--raiseerr', 'upgrade', 'head']) + engine = await gino.create_engine(DATABASE_URL) + db.bind = engine + except Exception: + logger.exception("Failed to connect to postgreSQL server", exc_info=True) + return + logger.info("Starting Kurisu on commit %s on branch %s", commit, branch) + bot = Kurisu(command_prefix=('.', '!'), description="Kurisu, the bot for Nintendo Homebrew!", commit=commit, + branch=branch) + bot.engine = engine + await bot.start(TOKEN) if __name__ == '__main__': - exit(main()) + loop = asyncio.get_event_loop() + loop.run_until_complete(startup()) diff --git a/migrations/env.py b/migrations/env.py index b885d5c5d..9f38c8384 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -38,7 +38,7 @@ def get_env(name: str): # Interpret the config file for Python logging. # This line sets up loggers basically. -fileConfig(config.config_file_name) +#fileConfig(config.config_file_name, disable_existing_loggers=False) config.set_main_option("sqlalchemy.url", DATABASE_URL) # add your model's MetaData object here diff --git a/utils/manager.py b/utils/manager.py index e9fb3743f..0639f46b4 100644 --- a/utils/manager.py +++ b/utils/manager.py @@ -16,7 +16,6 @@ async def load(self): for entry in await self.fetch_by_kind(kind=kind): self.filter[kind].append(entry.word) self.word_exp[entry.word] = re.compile(r"[ *_\-~]*".join(list(entry.word))) - print("Loaded word filter") async def add(self, word: str, kind: str) -> FilteredWord: entry = await FilteredWord.create(word=word, kind=kind) @@ -54,7 +53,6 @@ async def load(self): self.filter[kind].append((entry.word, entry.threshold)) if entry.whitelist: self.whitelist.append(entry.word) - print("Loaded levenshtein filter") async def add(self, word: str, kind: str, threshold: int, whitelist: bool) -> LevenshteinWord: entry = await LevenshteinWord.create(word=word, kind=kind, threshold=threshold, whitelist=whitelist) diff --git a/utils/utils.py b/utils/utils.py index a6d032f65..7e5ec1239 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -1,6 +1,7 @@ import discord import random import re +import traceback from discord.ext import commands @@ -78,9 +79,11 @@ def parse_time(time_string) -> int: def create_error_embed(ctx, exc) -> discord.Embed: - embed = discord.Embed(title=f"Unexpected exception in command {ctx.command}", color=0x2F3136) - embed.add_field(name=f"{exc.__class__.__name__} Exception ", value=exc, inline=False) - embed.add_field(name="Information", value=f"channel: {ctx.channel.mention if isinstance(ctx.channel, discord.TextChannel) else 'Direct Message'}\ncommand: {ctx.command}\nmessage: {ctx.message.content}\nuser: {ctx.author.mention}", inline=False) + embed = discord.Embed(title=f"Unexpected exception in command {ctx.command}", color=0xe50730) + trace = "".join(traceback.format_exception(etype=None, value=exc, tb=exc.__traceback__)) + embed.description = f'```py\n{trace}```' + embed.add_field(name="Exception Type", value=exc.__class__.__name__) + embed.add_field(name="Information", value=f"channel: {ctx.channel.mention if isinstance(ctx.channel, discord.TextChannel) else 'Direct Message'}\ncommand: {ctx.command}\nmessage: {ctx.message.content}\nauthor: {ctx.author.mention}", inline=False) return embed