fix: log errors on startup (#412)

* fix: log errors on startup - Add a basic startup logger to log any errors that occur during startup (YAML errors, validation errors, database errors, or any unknown errors). This log file will always be in the current working directory, as there is a possibility that the manager has not been set up yet. - Handle all posible cases of an invalid yaml file * refactor: update validation error footer Co-authored-by: Jacob <[email protected]> * refactor: remove console_debug_var * refactor: ignore `--console-log-level` unless `--no-ui` was used * refactor: simplify debug logic * docs: update `console_log_level` * refactor: only count running in IDE as DEBUG --------- Co-authored-by: Jacob <[email protected]>
jbsparrow · Dec 30, 2024 · ac402a7 · ac402a7
1 parent 67a1aee
commit ac402a7
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 109 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,7 +4,7 @@
 /AppData/
 /Downloads/
 /Old Files/
-cyberdrop_dl_debug.log
+*.log
 .coverage
 
 # Python cache

diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.py
@@ -3,7 +3,6 @@
 import asyncio
 import contextlib
 import logging
-import os
 import sys
 from functools import wraps
 from pathlib import Path
@@ -18,33 +17,38 @@
 from cyberdrop_dl.managers.manager import Manager
 from cyberdrop_dl.scraper.scraper import ScrapeMapper
 from cyberdrop_dl.ui.program_ui import ProgramUI
+from cyberdrop_dl.utils import constants
 from cyberdrop_dl.utils.apprise import send_apprise_notifications
-from cyberdrop_dl.utils.logger import RedactedConsole, log, log_spacer, log_with_color, print_to_console
+from cyberdrop_dl.utils.logger import RedactedConsole, log, log_spacer, log_with_color
 from cyberdrop_dl.utils.sorting import Sorter
 from cyberdrop_dl.utils.utilities import check_latest_pypi, check_partials_and_empty_folders, send_webhook_message
 from cyberdrop_dl.utils.yaml import handle_validation_error
 
 if TYPE_CHECKING:
     from collections.abc import Callable
 
+logger_startup = logging.getLogger("cyberdrop_dl_startup")
+
 
 def startup() -> Manager:
     """Starts the program and returns the manager.
 
     This will also run the UI for the program
     After this function returns, the manager will be ready to use and scraping / downloading can begin.
     """
+    setup_startup_logger()
     try:
         manager = Manager()
         manager.startup()
         if manager.parsed_args.cli_only_args.multiconfig:
+            logger_startup.info("validating all configs, please wait...")
             manager.validate_all_configs()
 
         if not manager.parsed_args.cli_only_args.download:
             ProgramUI(manager)
 
     except InvalidYamlError as e:
-        print_to_console(e.message, error=True)
+        logger_startup.error(e.message)
         sys.exit(1)
 
     except ValidationError as e:
@@ -56,9 +60,14 @@ def startup() -> Manager:
         handle_validation_error(e, sources=sources)
 
     except KeyboardInterrupt:
-        print_to_console("Exiting...")
+        logger_startup.info("Exiting...")
         sys.exit(0)
 
+    except Exception:
+        msg = "An error occurred, please report this to the developer with your logs file:"
+        logger_startup.exception(msg)
+        sys.exit(1)
+
     else:
         return manager
 
@@ -96,45 +105,49 @@ async def post_runtime(manager: Manager) -> None:
         await manager.log_manager.update_last_forum_post()
 
 
-def setup_debug_logger(manager: Manager) -> Path | None:
-    logger_debug = logging.getLogger("cyberdrop_dl_debug")
-    debug_log_file_path = None
-    running_in_IDE = os.getenv("PYCHARM_HOSTED") or os.getenv("TERM_PROGRAM") == "vscode"
-    from cyberdrop_dl.utils import constants
+def setup_startup_logger() -> None:
+    logger_startup.setLevel(10)
 
-    if running_in_IDE or manager.config_manager.settings_data.runtime_options.log_level == -1:
-        manager.config_manager.settings_data.runtime_options.log_level = 10
-        constants.DEBUG_VAR = True
+    rich_file_handler = RichHandler(
+        **constants.RICH_HANDLER_CONFIG,
+        console=RedactedConsole(
+            file=Path().cwd().joinpath("downloader.log").open("w", encoding="utf8"),
+            width=constants.DEFAULT_CONSOLE_WIDTH,
+        ),
+        level=10,
+    )
+    rich_handler = RichHandler(**(constants.RICH_HANDLER_CONFIG | {"show_time": False}), level=10)
+    logger_startup.addHandler(rich_file_handler)
+    logger_startup.addHandler(rich_handler)
 
-    if running_in_IDE or manager.config_manager.settings_data.runtime_options.console_log_level == -1:
-        constants.CONSOLE_DEBUG_VAR = True
 
-    if constants.DEBUG_VAR:
-        logger_debug.setLevel(manager.config_manager.settings_data.runtime_options.log_level)
-        debug_log_file_path = Path(__file__).parent / "cyberdrop_dl_debug.log"
-        if running_in_IDE:
-            debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"
+def setup_debug_logger(manager: Manager) -> Path | None:
+    if not constants.DEBUG_VAR:
+        return None
 
-        rich_file_handler_debug = RichHandler(
-            **constants.RICH_HANDLER_DEBUG_CONFIG,
-            console=Console(
-                file=debug_log_file_path.open("w", encoding="utf8"),
-                width=manager.config_manager.settings_data.logs.log_line_width,
-            ),
-            level=manager.config_manager.settings_data.runtime_options.log_level,
-        )
+    logger_debug = logging.getLogger("cyberdrop_dl_debug")
+    manager.config_manager.settings_data.runtime_options.log_level = 10
+    logger_debug.setLevel(manager.config_manager.settings_data.runtime_options.log_level)
+    debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"
+
+    rich_file_handler_debug = RichHandler(
+        **constants.RICH_HANDLER_DEBUG_CONFIG,
+        console=Console(
+            file=debug_log_file_path.open("w", encoding="utf8"),
+            width=manager.config_manager.settings_data.logs.log_line_width,
+        ),
+        level=manager.config_manager.settings_data.runtime_options.log_level,
+    )
 
-        logger_debug.addHandler(rich_file_handler_debug)
-        # aiosqlite_log = logging.getLogger("aiosqlite")
-        # aiosqlite_log.setLevel(manager.config_manager.settings_data.runtime_options.log_level)
-        # aiosqlite_log.addHandler(file_handler_debug)
+    logger_debug.addHandler(rich_file_handler_debug)
+    # aiosqlite_log = logging.getLogger("aiosqlite")
+    # aiosqlite_log.setLevel(manager.config_manager.settings_data.runtime_options.log_level)
+    # aiosqlite_log.addHandler(file_handler_debug)
 
-    return debug_log_file_path.resolve() if debug_log_file_path else None
+    return debug_log_file_path.resolve()
 
 
 def setup_logger(manager: Manager, config_name: str) -> None:
-    from cyberdrop_dl.utils import constants
-
     logger = logging.getLogger("cyberdrop_dl")
     if manager.multiconfig:
         if len(logger.handlers) > 0:
@@ -148,9 +161,6 @@ def setup_logger(manager: Manager, config_name: str) -> None:
 
     logger.setLevel(manager.config_manager.settings_data.runtime_options.log_level)
 
-    if constants.DEBUG_VAR:
-        manager.config_manager.settings_data.runtime_options.log_level = 10
-
     rich_file_handler = RichHandler(
         **constants.RICH_HANDLER_CONFIG,
         console=RedactedConsole(
@@ -160,8 +170,17 @@ def setup_logger(manager: Manager, config_name: str) -> None:
         level=manager.config_manager.settings_data.runtime_options.log_level,
     )
 
+    if manager.parsed_args.cli_only_args.no_ui:
+        constants.CONSOLE_LEVEL = manager.config_manager.settings_data.runtime_options.console_log_level
+
+    rich_handler = RichHandler(
+        **(constants.RICH_HANDLER_CONFIG | {"show_time": False}),
+        console=Console(),
+        level=constants.CONSOLE_LEVEL,
+    )
+
     logger.addHandler(rich_file_handler)
-    constants.CONSOLE_LEVEL = manager.config_manager.settings_data.runtime_options.console_log_level
+    logger.addHandler(rich_handler)
 
 
 def ui_error_handling_wrapper(func: Callable) -> None:
@@ -235,7 +254,7 @@ def main() -> None:
             asyncio.run(director(manager))
             exit_code = 0
         except KeyboardInterrupt:
-            print_to_console("Trying to Exit ...")
+            logger_startup.info("Trying to Exit ...")
         finally:
             asyncio.run(manager.close())
     loop.close()

diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
@@ -18,9 +18,10 @@
 from cyberdrop_dl.managers.path_manager import PathManager
 from cyberdrop_dl.managers.progress_manager import ProgressManager
 from cyberdrop_dl.managers.realdebrid_manager import RealDebridManager
+from cyberdrop_dl.utils import constants
 from cyberdrop_dl.utils.args import ParsedArgs
 from cyberdrop_dl.utils.data_enums_classes.supported_domains import SUPPORTED_FORUMS
-from cyberdrop_dl.utils.logger import log, print_to_console
+from cyberdrop_dl.utils.logger import log
 from cyberdrop_dl.utils.transfer.db_setup import TransitionManager
 
 if TYPE_CHECKING:
@@ -62,6 +63,7 @@ def __init__(self) -> None:
 
     def startup(self) -> None:
         """Startup process for the manager."""
+
         if isinstance(self.parsed_args, Field):
             self.parsed_args = ParsedArgs.parse_args()
 
@@ -118,8 +120,6 @@ async def async_startup(self) -> None:
             self.real_debrid_manager = RealDebridManager(self)
         await self.async_db_hash_startup()
 
-        from cyberdrop_dl.utils import constants
-
         constants.MAX_NAME_LENGTHS["FILE"] = self.config_manager.global_settings_data.general.max_file_name_length
         constants.MAX_NAME_LENGTHS["FOLDER"] = self.config_manager.global_settings_data.general.max_folder_name_length
 
@@ -219,7 +219,6 @@ async def close(self) -> None:
         self.hash_manager: HashManager = field(init=False)
 
     def validate_all_configs(self) -> None:
-        print_to_console("validating all configs, please wait...")
         all_configs = self.config_manager.get_configs()
         all_configs.sort()
         if not all_configs:

diff --git a/cyberdrop_dl/scraper/__init__.py b/cyberdrop_dl/scraper/__init__.py
@@ -1,8 +1,10 @@
 # ruff: noqa: F401
 from __future__ import annotations
 
+import os
 from typing import TYPE_CHECKING
 
+from cyberdrop_dl import __version__ as current_version
 from cyberdrop_dl.scraper.crawlers.bunkrr_crawler import BunkrrCrawler
 from cyberdrop_dl.scraper.crawlers.celebforum_crawler import CelebForumCrawler
 from cyberdrop_dl.scraper.crawlers.chevereto_crawler import CheveretoCrawler
@@ -44,9 +46,19 @@
 from cyberdrop_dl.scraper.crawlers.xbunker_crawler import XBunkerCrawler
 from cyberdrop_dl.scraper.crawlers.xbunkr_crawler import XBunkrCrawler
 from cyberdrop_dl.scraper.crawlers.xxxbunker_crawler import XXXBunkerCrawler
+from cyberdrop_dl.utils import constants
 
 if TYPE_CHECKING:
     from cyberdrop_dl.scraper.crawler import Crawler
 
-ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if "Crawler" in name}
+ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler")}
 DEBUG_CRAWLERS = {SimpCityCrawler}
+CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS
+
+constants.RUNNING_PRERELEASE = next((tag for tag in constants.PRERELEASE_TAGS if tag in current_version), False)
+RUNNING_IN_IDE = os.getenv("PYCHARM_HOSTED") or os.getenv("TERM_PROGRAM") == "vscode"
+if constants.RUNNING_PRERELEASE or RUNNING_IN_IDE:
+    CRAWLERS = ALL_CRAWLERS
+
+if RUNNING_IN_IDE:
+    constants.DEBUG_VAR = True
diff --git a/cyberdrop_dl/scraper/scraper.py b/cyberdrop_dl/scraper/scraper.py
@@ -10,10 +10,9 @@
 import arrow
 from yarl import URL
 
-from cyberdrop_dl import __version__ as current_version
 from cyberdrop_dl.clients.errors import JDownloaderError
 from cyberdrop_dl.downloader.downloader import Downloader
-from cyberdrop_dl.scraper import ALL_CRAWLERS, DEBUG_CRAWLERS
+from cyberdrop_dl.scraper import CRAWLERS
 from cyberdrop_dl.scraper.filters import (
     has_valid_extension,
     is_in_domain_list,
@@ -22,7 +21,7 @@
     remove_trailing_slash,
 )
 from cyberdrop_dl.scraper.jdownloader import JDownloader
-from cyberdrop_dl.utils.constants import BLOCKED_DOMAINS, PRERELEASE_TAGS, REGEX_LINKS
+from cyberdrop_dl.utils.constants import BLOCKED_DOMAINS, REGEX_LINKS
 from cyberdrop_dl.utils.data_enums_classes.url_objects import MediaItem, ScrapeItem
 from cyberdrop_dl.utils.logger import log
 from cyberdrop_dl.utils.utilities import get_download_path, get_filename_and_ext
@@ -47,12 +46,7 @@ def __init__(self, manager: Manager) -> None:
 
     def start_scrapers(self) -> None:
         """Starts all scrapers."""
-        crawlers = ALL_CRAWLERS
-        is_testing = next((tag for tag in PRERELEASE_TAGS if tag in current_version), False)
-        if not is_testing:
-            crawlers -= DEBUG_CRAWLERS
-
-        for crawler in crawlers:
+        for crawler in CRAWLERS:
             if not crawler.SUPPORTED_SITES:
                 site_crawler = crawler(self.manager)
                 self.existing_crawlers[site_crawler.domain] = site_crawler

diff --git a/cyberdrop_dl/utils/constants.py b/cyberdrop_dl/utils/constants.py
@@ -9,7 +9,7 @@
 MAX_NAME_LENGTHS = {"FILE": 95, "FOLDER": 60}
 DEFAULT_CONSOLE_WIDTH = 240
 DEBUG_VAR = False
-CONSOLE_DEBUG_VAR = False
+RUNNING_PRERELEASE = False
 CSV_DELIMITER = ","
 LOG_OUTPUT_TEXT = Text("")
 RICH_HANDLER_CONFIG = {"show_time": True, "rich_tracebacks": True, "tracebacks_show_locals": False}

diff --git a/cyberdrop_dl/utils/data_enums_classes/supported_domains.py b/cyberdrop_dl/utils/data_enums_classes/supported_domains.py
@@ -2,21 +2,14 @@
 
 from typing import TYPE_CHECKING
 
-from cyberdrop_dl import __version__ as current_version
-from cyberdrop_dl.scraper import ALL_CRAWLERS, DEBUG_CRAWLERS
+from cyberdrop_dl.scraper import CRAWLERS
 from cyberdrop_dl.scraper.crawlers.xenforo_crawler import XenforoCrawler
-from cyberdrop_dl.utils.constants import PRERELEASE_TAGS
 
 if TYPE_CHECKING:
     from cyberdrop_dl.scraper.crawler import Crawler
 
-crawlers = ALL_CRAWLERS
-is_testing = next((tag for tag in PRERELEASE_TAGS if tag in current_version), False)
-if not is_testing:
-    crawlers -= DEBUG_CRAWLERS
-
-forum_crawlers = {crawler for crawler in crawlers if issubclass(crawler, XenforoCrawler)}
-website_crawlers = crawlers - forum_crawlers
+forum_crawlers = {crawler for crawler in CRAWLERS if issubclass(crawler, XenforoCrawler)}
+website_crawlers = CRAWLERS - forum_crawlers
 
 
 def get_supported_sites_from(crawlers: set[type[Crawler]]) -> dict[str, str]:

diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
@@ -22,16 +22,10 @@ def _render_buffer(self, buffer) -> str:
         return _redact_message(output)
 
 
-def print_to_console(text: Text | str, *, error: bool = False, **kwargs) -> None:
-    msg = (ERROR_PREFIX + text) if error else text
-    console.print(msg, **kwargs)
-
-
 def log(message: Exception | str, level: int = 10, *, sleep: int | None = None, **kwargs) -> None:
     """Simple logging function."""
     logger.log(level, message, **kwargs)
     log_debug(message, level, **kwargs)
-    log_debug_console(message, level, sleep=sleep)
 
 
 def log_debug(message: Exception | str, level: int = 10, **kwargs) -> None:
@@ -41,12 +35,6 @@ def log_debug(message: Exception | str, level: int = 10, **kwargs) -> None:
         logger_debug.log(level, message.encode("ascii", "ignore").decode("ascii"), **kwargs)
 
 
-def log_debug_console(message: Exception | str, level: int, sleep: int | None = None) -> None:
-    if constants.CONSOLE_DEBUG_VAR:
-        message = str(message)
-        _log_to_console(level, message.encode("ascii", "ignore").decode("ascii"), sleep=sleep)
-
-
 def log_with_color(message: str, style: str, level: int, show_in_stats: bool = True, **kwargs) -> None:
     """Simple logging function with color."""
     log(message, level, **kwargs)
@@ -66,12 +54,6 @@ def log_spacer(level: int, char: str = "-", *, log_to_console: bool = True, log_
     constants.LOG_OUTPUT_TEXT.append("\n", style="black")
 
 
-def _log_to_console(level: int, record: str, *_, **__) -> None:
-    level = level or 10
-    if level >= constants.CONSOLE_LEVEL:
-        console.log(record)
-
-
 def _redact_message(message: Exception | Text | str) -> str:
     redacted = str(message)
     separators = ["\\", "\\\\", "/"]