From 8c16998ec634a64f21763b06919fa2d4d2da7695 Mon Sep 17 00:00:00 2001 From: Revulate Date: Mon, 14 Oct 2024 18:58:09 -0700 Subject: [PATCH] modified: bot.py modified: cogs/dvp.py modified: cogs/rate.py modified: logger.py modified: twitch_helix_client.py --- bot.py | 29 ++++- cogs/dvp.py | 264 +++++++++++------------------------------ cogs/rate.py | 15 +-- logger.py | 34 +++++- twitch_helix_client.py | 7 +- 5 files changed, 135 insertions(+), 214 deletions(-) diff --git a/bot.py b/bot.py index a8028fd..ad14a56 100644 --- a/bot.py +++ b/bot.py @@ -75,6 +75,9 @@ def __init__(self): self.twitch_api.refresh_token = self.refresh_token_string self.logger.info("TwitchAPI instance created and tokens saved") + self._connection_retries = 0 + self._max_retries = 5 + async def event_ready(self): self.logger.info(f"Logged in as | {self.nick}") self.load_tokens_from_file() @@ -102,6 +105,31 @@ async def event_ready(self): ) self.logger.info(f"Token expiry: {self.twitch_api.token_expiry}") + async def start(self): + while self._connection_retries < self._max_retries: + try: + await super().start() + break + except Exception as e: + self._connection_retries += 1 + self.logger.error(f"Connection attempt {self._connection_retries} failed: {e}") + if self._connection_retries < self._max_retries: + await asyncio.sleep(5 * self._connection_retries) # Exponential backoff + else: + self.logger.error("Max retries reached. Unable to connect.") + raise + + async def close(self): + try: + if self._connection and hasattr(self._connection, "_close"): + await self._connection._close() + if hasattr(self, "_http") and self._http: + await self._http.close() + except Exception as e: + self.logger.error(f"Error during close: {e}") + finally: + await super().close() + async def check_token_regularly(self): while True: await asyncio.sleep(3600) # Check every hour @@ -218,7 +246,6 @@ async def event_error(self, error: Exception, data: str = None): async def main(): - load_dotenv() bot = TwitchBot() while True: diff --git a/cogs/dvp.py b/cogs/dvp.py index 25695e1..f3cdc2e 100644 --- a/cogs/dvp.py +++ b/cogs/dvp.py @@ -1,3 +1,5 @@ +# cogs/dvp.py + import asyncio import aiosqlite from twitchio.ext import commands @@ -8,8 +10,9 @@ from google.oauth2.service_account import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError -import re from playwright.async_api import async_playwright +from tenacity import retry, stop_after_attempt, wait_exponential +import re # Added for regex operations class DVP(commands.Cog): @@ -17,17 +20,13 @@ def __init__(self, bot): self.bot = bot self.db_path = "vulpes_games.db" self.channel_name = "vulpeshd" - self.client_id = os.getenv("TWITCH_CLIENT_ID") - self.client_secret = os.getenv("TWITCH_CLIENT_SECRET") - self.logger = bot.logger.getChild("dvp") + self.logger = logging.getLogger("twitch_bot.cogs.dvp") # Use child logger self.logger.setLevel(logging.DEBUG) - self.update_task = None - self.update_recent_games_task = None + self.update_scrape_task = None self.sheet_id = os.getenv("GOOGLE_SHEET_ID") self.creds_file = os.getenv("GOOGLE_CREDENTIALS_FILE") self.db_initialized = asyncio.Event() self.last_scrape_time = None - self.twitch_api = bot.twitch_api if not self.sheet_id: raise ValueError("GOOGLE_SHEET_ID is not set in the environment variables") @@ -70,8 +69,7 @@ async def initialize_cog(self): await self.setup_database() await self.load_last_scrape_time() await self.initialize_data() - self.update_task = self.bot.loop.create_task(self.periodic_update()) - self.update_recent_games_task = self.bot.loop.create_task(self.periodic_recent_games_update()) + self.update_scrape_task = self.bot.loop.create_task(self.periodic_scrape_update()) self.logger.info("DVP cog initialized successfully") async def setup_database(self): @@ -88,17 +86,6 @@ async def setup_database(self): ) """ ) - await db.execute( - """ - CREATE TABLE IF NOT EXISTS streams ( - id TEXT PRIMARY KEY, - game_id TEXT, - game_name TEXT, - started_at TEXT, - duration INTEGER - ) - """ - ) await db.execute( """ CREATE TABLE IF NOT EXISTS metadata ( @@ -118,8 +105,12 @@ async def load_last_scrape_time(self): async with db.execute("SELECT value FROM metadata WHERE key = 'last_scrape_time'") as cursor: result = await cursor.fetchone() if result: - self.last_scrape_time = datetime.fromisoformat(result[0]) - self.logger.info(f"Last scrape time loaded: {self.last_scrape_time}") + try: + self.last_scrape_time = datetime.fromisoformat(result[0]) + self.logger.info(f"Last scrape time loaded: {self.last_scrape_time}") + except ValueError as ve: + self.logger.error(f"Invalid datetime format in metadata: {result[0]}", exc_info=True) + self.last_scrape_time = None async def save_last_scrape_time(self): current_time = datetime.now(timezone.utc) @@ -141,8 +132,6 @@ async def initialize_data(self): await self.save_last_scrape_time() else: self.logger.info("Skipping web scraping, using existing data") - - await self.fetch_and_process_recent_videos() await self.update_initials_mapping() self.db_initialized.set() self.logger.info("Data initialization completed successfully.") @@ -172,17 +161,14 @@ async def scrape_initial_data(self): columns = await row.query_selector_all("td") if len(columns) >= 7: name = (await columns[1].inner_text()).strip() - - time_played_element = await columns[2].query_selector("span") - if time_played_element: - time_played_str = (await time_played_element.inner_text()).strip() - else: - time_played_str = (await columns[2].inner_text()).strip() - self.logger.debug(f"Scraped time_played_str for '{name}': '{time_played_str}'") - time_played_str, time_played = self.parse_time(time_played_str) - + time_played_str = (await columns[2].inner_text()).strip() + _, time_played = self.parse_time(time_played_str) last_played_str = (await columns[6].inner_text()).strip() - last_played = datetime.strptime(last_played_str, "%d/%b/%Y").date() + try: + last_played = datetime.strptime(last_played_str, "%d/%b/%Y").date() + except ValueError as ve: + self.logger.error(f"Error parsing date '{last_played_str}': {ve}", exc_info=True) + last_played = datetime.now(timezone.utc).date() await db.execute( """ INSERT INTO games (name, time_played, last_played) @@ -200,110 +186,34 @@ async def scrape_initial_data(self): except Exception as e: self.logger.error(f"Error during data scraping: {e}", exc_info=True) - async def fetch_and_process_recent_videos(self): - user_id = await self.twitch_api.get_user_id(self.channel_name) - if not user_id: - self.logger.error(f"Could not find user ID for {self.channel_name}") - return - - videos = await self.twitch_api.fetch_recent_videos(user_id) - await self.process_video_data(videos) - - async def process_video_data(self, videos): - self.logger.info(f"Processing {len(videos)} videos from Twitch API") - game_playtimes = {} - - async with aiosqlite.connect(self.db_path) as db: - await db.execute("DELETE FROM streams") - - for video in videos: - video_id = video["id"] - game_name = video.get("game_name") - if not game_name or game_name.lower() == "unknown": - continue - - game_id = video.get("game_id", "") - created_at = video["created_at"] - duration_str = video["duration"] - duration = self.parse_duration(duration_str) - - self.logger.debug(f"Video {video_id}: Game: {game_name}, Duration: {duration} seconds") - - if game_name in game_playtimes: - game_playtimes[game_name] += duration - else: - game_playtimes[game_name] = duration - - await db.execute( - """ - INSERT OR REPLACE INTO streams - (id, game_id, game_name, started_at, duration) - VALUES (?, ?, ?, ?, ?) - """, - (video_id, game_id, game_name, created_at, duration), - ) - - await db.commit() - - # Update the games table with the new playtimes - async with aiosqlite.connect(self.db_path) as db: - for game_name, total_duration in game_playtimes.items(): - total_minutes = total_duration // 60 - last_played = datetime.now(timezone.utc).date() - self.logger.debug( - f"Updating game: {game_name}, Total duration: {total_duration} seconds, Total minutes: {total_minutes}" - ) - await db.execute( - """ - INSERT INTO games (name, time_played, last_played) - VALUES (?, ?, ?) - ON CONFLICT(name) DO UPDATE SET - time_played = excluded.time_played, - last_played = excluded.last_played - """, - (game_name, total_minutes, last_played), - ) - await db.commit() - - for game_name in ["FINAL FANTASY VII REBIRTH", "FINAL FANTASY XVI", "ELDEN RING"]: - total_minutes = game_playtimes.get(game_name, 0) // 60 - self.logger.info(f"Total playtime for {game_name}: {self.format_playtime(total_minutes)}") - - self.logger.info("Video data processing and game playtime update completed.") - - def parse_duration(self, duration_str): - total_seconds = 0 - time_parts = duration_str.split("h") - if len(time_parts) > 1: - hours = int(time_parts[0]) - total_seconds += hours * 3600 - duration_str = time_parts[1] - - time_parts = duration_str.split("m") - if len(time_parts) > 1: - minutes = int(time_parts[0]) - total_seconds += minutes * 60 - duration_str = time_parts[1] - - time_parts = duration_str.split("s") - if len(time_parts) > 1: - seconds = int(time_parts[0]) - total_seconds += seconds - - return total_seconds - def parse_time(self, time_str): + """ + Parses the time played string and converts it to total minutes. + Expected formats: + - "596\n20.5%" -> 596 minutes + - "0.5\n0%" -> 0.5 hours (30 minutes) + - "1 day 2 hours" -> 1560 minutes + """ total_minutes = 0 - time_str = time_str.replace(",", "").strip() + # Remove commas, strip spaces, and split by newline to remove percentage + time_str = time_str.replace(",", "").strip().split("\n")[0] + # Remove any '%' symbols + time_str = time_str.replace("%", "") try: if "." in time_str: + # Assume the value before the newline is in hours hours = float(time_str) total_minutes = int(hours * 60) else: parts = time_str.split() i = 0 while i < len(parts): - value = float(parts[i]) + # Use regex to extract numeric value + value_match = re.match(r"(\d+(\.\d+)?)", parts[i]) + if not value_match: + self.logger.error(f"Invalid numeric value in time string '{time_str}'") + break + value = float(value_match.group(1)) if i + 1 < len(parts): unit = parts[i + 1].rstrip("s").lower() i += 2 @@ -336,20 +246,6 @@ def format_playtime(self, total_minutes): time_played = f"{int(minutes)}m" return time_played - async def update_game_data(self, game_name, time_played): - async with aiosqlite.connect(self.db_path) as db: - await db.execute( - """ - INSERT INTO games (name, time_played, last_played) - VALUES (?, ?, ?) - ON CONFLICT(name) DO UPDATE SET - time_played = time_played + excluded.time_played, - last_played = excluded.last_played - """, - (game_name, time_played, datetime.now(timezone.utc).date()), - ) - await db.commit() - @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)) async def update_google_sheet(self): creds = Credentials.from_service_account_file( @@ -364,24 +260,21 @@ async def update_google_sheet(self): rows = await cursor.fetchall() # Prepare data for the sheet - headers = ["Game Image", "Game Name", "Time Played", "Last Played"] + headers = ["Game Name", "Time Played", "Last Played"] data = [headers] for row in rows: name, minutes, last_played = row - game_image_url = await self.twitch_api.get_game_image_url(name) - # Format the time played into days, hours, minutes time_played = self.format_playtime(minutes) - # Format the last played date - last_played_date = datetime.strptime(str(last_played), "%Y-%m-%d") - last_played_formatted = last_played_date.strftime("%B %d, %Y") - - # Prepare the image formula - img_formula = f'=IMAGE("{game_image_url}")' if game_image_url else "" - - data.append([img_formula, name, time_played, last_played_formatted]) + try: + last_played_date = datetime.strptime(str(last_played), "%Y-%m-%d") + last_played_formatted = last_played_date.strftime("%B %d, %Y") + except ValueError as ve: + self.logger.error(f"Error formatting last played date '{last_played}': {ve}", exc_info=True) + last_played_formatted = str(last_played) + data.append([name, time_played, last_played_formatted]) # Update the data starting from cell A3 body = {"values": data} @@ -390,7 +283,7 @@ async def update_google_sheet(self): # Clear existing data from A3 onwards service.spreadsheets().values().clear( spreadsheetId=self.sheet_id, - range="A3:D1000", # Adjust the end row as needed + range="A3:C1000", # Adjust the end row as needed ).execute() # Write the data @@ -464,64 +357,34 @@ async def apply_sheet_formatting(self, service, data_row_count): self.logger.info("Applied formatting to the Google Sheet.") except HttpError as e: self.logger.error(f"An error occurred while applying formatting: {e}") + except Exception as ex: + self.logger.error(f"Unexpected error during sheet formatting: {ex}", exc_info=True) async def get_sheet_id(self, service, spreadsheet_id): try: sheet_metadata = service.spreadsheets().get(spreadsheetId=spreadsheet_id).execute() sheets = sheet_metadata.get("sheets", "") + if not sheets: + self.logger.error(f"No sheets found in spreadsheet ID '{spreadsheet_id}'.") + return 0 sheet_id = sheets[0].get("properties", {}).get("sheetId", 0) return sheet_id except Exception as e: self.logger.error(f"Error retrieving sheet ID: {e}", exc_info=True) return 0 - async def periodic_update(self): - while True: - try: - current_game = await self.twitch_api.get_channel_games(self.channel_name) - if current_game: - await self.update_game_data(current_game, 5) # Update every 5 minutes - except Exception as e: - self.logger.error(f"Error during periodic update: {e}", exc_info=True) - await asyncio.sleep(300) # Sleep for 5 minutes - - async def periodic_recent_games_update(self): + async def periodic_scrape_update(self): while True: try: - await self.fetch_and_process_recent_videos() - await self.update_game_playtime() + self.logger.info("Starting periodic web scraping update.") + await self.scrape_initial_data() + await self.save_last_scrape_time() + await self.update_initials_mapping() await self.update_google_sheet() + self.logger.info("Periodic web scraping update completed.") except Exception as e: - self.logger.error(f"Error during periodic recent games update: {e}", exc_info=True) - await asyncio.sleep(86400) # Update daily - - async def update_game_playtime(self): - async with aiosqlite.connect(self.db_path) as db: - # Sum up the durations for each game - async with db.execute( - """ - SELECT game_name, SUM(duration) - FROM streams - GROUP BY game_name - """ - ) as cursor: - game_playtimes = await cursor.fetchall() - - for game_name, total_duration in game_playtimes: - # Convert total_duration from seconds to minutes - total_minutes = total_duration // 60 - last_played = datetime.now(timezone.utc).date() - await db.execute( - """ - INSERT INTO games (name, time_played, last_played) - VALUES (?, ?, ?) - ON CONFLICT(name) DO UPDATE SET - time_played = excluded.time_played, - last_played = excluded.last_played - """, - (game_name, total_minutes, last_played), - ) - await db.commit() + self.logger.error(f"Error during periodic web scraping update: {e}", exc_info=True) + await asyncio.sleep(86400) # Run once every 24 hours async def update_initials_mapping(self): # Use only predefined abbreviations @@ -594,7 +457,12 @@ async def did_vulpes_play_it(self, ctx: commands.Context, *, game_name: str): if result: time_played, last_played = result formatted_time = self.format_playtime(time_played) - last_played_formatted = datetime.strptime(str(last_played), "%Y-%m-%d").strftime("%B %d, %Y") + try: + last_played_date = datetime.strptime(str(last_played), "%Y-%m-%d") + last_played_formatted = last_played_date.strftime("%B %d, %Y") + except ValueError as ve: + self.logger.error(f"Error formatting last played date '{last_played}': {ve}", exc_info=True) + last_played_formatted = str(last_played) await ctx.send( f"@{ctx.author.name}, Vulpes played {game_name_to_search} for {formatted_time}. Last played on {last_played_formatted}." ) diff --git a/cogs/rate.py b/cogs/rate.py index 3339427..de757fe 100644 --- a/cogs/rate.py +++ b/cogs/rate.py @@ -1,16 +1,17 @@ -# rate.py +# cogs/rate.py + import random from twitchio.ext import commands -from logger import setup_logger # Import the centralized logger -from utils import split_message # Import the shared split_message function +import logging +from utils import split_message # Ensure this is correctly implemented class Rate(commands.Cog): - """Cog for handling various rate-based commands like 'cute', 'gay', 'iq', etc.'""" + """Cog for handling various rate-based commands like 'cute', 'gay', 'iq', etc.""" def __init__(self, bot): self.bot = bot - self.logger = setup_logger("twitch_bot.cogs.rate") # Reuse the centralized logger + self.logger = logging.getLogger("twitch_bot.cogs.rate") # Use child logger def get_mentioned_user(self, ctx: commands.Context, mentioned_user: str = None): """Helper method to extract a mentioned user or default to the command author.""" @@ -135,7 +136,7 @@ async def all_command(self, ctx: commands.Context, *, mentioned_user: str = None messages.append(f"{user} 's pp is {length_str} long and has a {girth_inches}in girth. BillyApprove") rating = random.randint(0, 10) - messages.append(f"{user} is a {rating}/10. {'CHUG' if rating > 5 else 'Hmm'}") + messages.append(f"I would give {user} a {rating}/10. {'CHUG' if rating > 5 else 'Hmm'}") horny_percentage = random.randint(0, 100) messages.append(f"{user} is {horny_percentage}% horny right now. {'HORNY' if horny_percentage > 50 else 'Hmm'}") @@ -144,7 +145,7 @@ async def all_command(self, ctx: commands.Context, *, mentioned_user: str = None iq_description = ( "thoughtless" if iq <= 50 - else "slowpoke" if iq <= 80 else "NPC" if iq <= 115 else "catNerd" if iq <= 199 else "BrainGalaxy" + else "a slowpoke" if iq <= 80 else "an NPC" if iq <= 115 else "catNerd" if iq <= 199 else "BrainGalaxy" ) messages.append(f"{user} has {iq} IQ. {iq_description}") diff --git a/logger.py b/logger.py index e028d4b..340dcab 100644 --- a/logger.py +++ b/logger.py @@ -39,10 +39,18 @@ def setup_logger(name="twitch_bot", level=None, log_file=None, max_bytes=None, b with _logger_setup_lock: if not logger.handlers: - logger.addHandler(create_console_handler(level)) - logger.addHandler(create_file_handler(log_file, max_bytes, backup_count, level)) + console_handler = create_console_handler(level) + file_handler = create_file_handler(log_file, max_bytes, backup_count, level) + handlers = [console_handler, file_handler] + if LOGDNA_KEY: - logger.addHandler(create_logdna_handler(level)) + logdna_handler = create_logdna_handler(level) + if logdna_handler: + handlers.append(logdna_handler) + + for handler in handlers: + logger.addHandler(handler) + logger.setLevel(level) return logger @@ -53,6 +61,16 @@ def create_console_handler(level): console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(level) console_handler.setFormatter(_formatter) + + # Explicitly set encoding to 'utf-8' to handle Unicode characters + if sys.stdout.encoding.lower() != "utf-8": + try: + console_handler.setEncoding("utf-8") + logging.info("Console handler encoding set to UTF-8.") + except AttributeError: + # For Python versions < 3.7 where setEncoding might not be available + logging.warning("Failed to set console handler encoding to UTF-8. Python version may not support it.") + return console_handler @@ -68,9 +86,13 @@ def create_logdna_handler(level): """Create a LogDNA handler for logging.""" if LOGDNA_KEY: logdna_options = {"app": "TwitchBot", "env": "production"} - logdna_handler = LogDNAHandler(LOGDNA_KEY, options=logdna_options) - logdna_handler.setLevel(level) - return logdna_handler + try: + logdna_handler = LogDNAHandler(LOGDNA_KEY, options=logdna_options) + logdna_handler.setLevel(level) + logdna_handler.setFormatter(_formatter) + return logdna_handler + except Exception as e: + print(f"Failed to create LogDNA handler: {e}") return None diff --git a/twitch_helix_client.py b/twitch_helix_client.py index fea3610..200859a 100644 --- a/twitch_helix_client.py +++ b/twitch_helix_client.py @@ -30,7 +30,7 @@ def set_session(self, session): """Set an external session for API requests.""" self.session = session - async def initialize_session(self): + async def ensure_session(self): if self.session is None or self.session.closed: self.session = aiohttp.ClientSession() @@ -120,7 +120,7 @@ async def ensure_token_valid(self): return True async def api_request(self, endpoint, params=None, method="GET", data=None): - await self.ensure_token_valid() + await self.ensure_session() url = f"{self.BASE_URL}/{endpoint}" headers = { @@ -245,3 +245,6 @@ async def get_game_image_url(self, game_name): except Exception as e: logger.error(f"Error fetching image URL for {game_name}: {e}") return "" + + async def close(self): + await self.close_session()