diff --git a/supervisor/addons/addon.py b/supervisor/addons/addon.py index f1680bf81b3..37c23a059da 100644 --- a/supervisor/addons/addon.py +++ b/supervisor/addons/addon.py @@ -81,7 +81,8 @@ from ..homeassistant.const import WSEvent, WSType from ..jobs.const import JobExecutionLimit from ..jobs.decorator import Job -from ..resolution.const import UnhealthyReason +from ..resolution.const import ContextType, IssueType, UnhealthyReason +from ..resolution.data import Issue from ..store.addon import AddonStore from ..utils import check_port from ..utils.apparmor import adjust_profile @@ -144,11 +145,19 @@ def __init__(self, coresys: CoreSys, slug: str): self._listeners: list[EventListener] = [] self._startup_event = asyncio.Event() self._startup_task: asyncio.Task | None = None + self._boot_failed_issue = Issue( + IssueType.BOOT_FAIL, ContextType.ADDON, reference=self.slug + ) def __repr__(self) -> str: """Return internal representation.""" return f"" + @property + def boot_failed_issue(self) -> Issue: + """Get issue used if start on boot failed.""" + return self._boot_failed_issue + @property def state(self) -> AddonState: """Return state of the add-on.""" @@ -166,6 +175,13 @@ def state(self, new_state: AddonState) -> None: if new_state == AddonState.STARTED or old_state == AddonState.STARTUP: self._startup_event.set() + # Dismiss boot failed issue if present and we started + if ( + new_state == AddonState.STARTED + and self.boot_failed_issue in self.sys_resolution.issues + ): + self.sys_resolution.dismiss_issue(self.boot_failed_issue) + self.sys_homeassistant.websocket.send_message( { ATTR_TYPE: WSType.SUPERVISOR_EVENT, @@ -322,6 +338,13 @@ def boot(self, value: AddonBoot) -> None: """Store user boot options.""" self.persist[ATTR_BOOT] = value + # Dismiss boot failed issue if present and boot at start disabled + if ( + value == AddonBoot.MANUAL + and self._boot_failed_issue in self.sys_resolution.issues + ): + self.sys_resolution.dismiss_issue(self._boot_failed_issue) + @property def auto_update(self) -> bool: """Return if auto update is enable.""" diff --git a/supervisor/addons/manager.py b/supervisor/addons/manager.py index 1f255414992..b5ceecd621b 100644 --- a/supervisor/addons/manager.py +++ b/supervisor/addons/manager.py @@ -7,24 +7,22 @@ import tarfile from typing import Union +from attr import evolve + from ..const import AddonBoot, AddonStartup, AddonState from ..coresys import CoreSys, CoreSysAttributes from ..exceptions import ( - AddonConfigurationError, AddonsError, AddonsJobError, AddonsNotSupportedError, CoreDNSError, - DockerAPIError, DockerError, - DockerNotFound, HassioError, HomeAssistantAPIError, ) from ..jobs.decorator import Job, JobCondition from ..resolution.const import ContextType, IssueType, SuggestionType from ..store.addon import AddonStore -from ..utils import check_exception_chain from ..utils.sentry import capture_exception from .addon import Addon from .const import ADDON_UPDATE_CONDITIONS @@ -118,15 +116,14 @@ async def boot(self, stage: AddonStartup) -> None: try: if start_task := await addon.start(): wait_boot.append(start_task) - except AddonsError as err: - # Check if there is an system/user issue - if check_exception_chain( - err, (DockerAPIError, DockerNotFound, AddonConfigurationError) - ): - addon.boot = AddonBoot.MANUAL - addon.save_persist() except HassioError: - pass # These are already handled + self.sys_resolution.add_issue( + evolve(addon.boot_failed_issue), + suggestions=[ + SuggestionType.EXECUTE_START, + SuggestionType.DISABLE_BOOT, + ], + ) else: continue @@ -135,6 +132,19 @@ async def boot(self, stage: AddonStartup) -> None: # Ignore exceptions from waiting for addon startup, addon errors handled elsewhere await asyncio.gather(*wait_boot, return_exceptions=True) + # After waiting for startup, create an issue for boot addons that are error or unknown state + # Ignore stopped as single shot addons can be run at boot and this is successful exit + # Timeout waiting for startup is not a failure, addon is probably just slow + for addon in tasks: + if addon.state in {AddonState.ERROR, AddonState.UNKNOWN}: + self.sys_resolution.add_issue( + evolve(addon.boot_failed_issue), + suggestions=[ + SuggestionType.EXECUTE_START, + SuggestionType.DISABLE_BOOT, + ], + ) + async def shutdown(self, stage: AddonStartup) -> None: """Shutdown addons.""" tasks: list[Addon] = [] diff --git a/supervisor/mounts/manager.py b/supervisor/mounts/manager.py index 332423f3daa..4bb1278cf57 100644 --- a/supervisor/mounts/manager.py +++ b/supervisor/mounts/manager.py @@ -6,6 +6,8 @@ import logging from pathlib import PurePath +from attr import evolve + from ..const import ATTR_NAME from ..coresys import CoreSys, CoreSysAttributes from ..dbus.const import UnitActiveState @@ -171,7 +173,7 @@ async def _mount_errors_to_issues( capture_exception(errors[i]) self.sys_resolution.add_issue( - mounts[i].failed_issue, + evolve(mounts[i].failed_issue), suggestions=[ SuggestionType.EXECUTE_RELOAD, SuggestionType.EXECUTE_REMOVE, diff --git a/supervisor/mounts/mount.py b/supervisor/mounts/mount.py index 5ee1f2fea09..97466580da9 100644 --- a/supervisor/mounts/mount.py +++ b/supervisor/mounts/mount.py @@ -68,6 +68,9 @@ def __init__(self, coresys: CoreSys, data: MountData) -> None: self._data: MountData = data self._unit: SystemdUnit | None = None self._state: UnitActiveState | None = None + self._failed_issue = Issue( + IssueType.MOUNT_FAILED, ContextType.MOUNT, reference=self.name + ) @classmethod def from_dict(cls, coresys: CoreSys, data: MountData) -> "Mount": @@ -162,7 +165,7 @@ def local_where(self) -> Path | None: @property def failed_issue(self) -> Issue: """Get issue used if this mount has failed.""" - return Issue(IssueType.MOUNT_FAILED, ContextType.MOUNT, reference=self.name) + return self._failed_issue async def is_mounted(self) -> bool: """Return true if successfully mounted and available.""" diff --git a/supervisor/resolution/const.py b/supervisor/resolution/const.py index face2e72f27..13b74c34044 100644 --- a/supervisor/resolution/const.py +++ b/supervisor/resolution/const.py @@ -71,6 +71,7 @@ class UnhealthyReason(StrEnum): class IssueType(StrEnum): """Issue type.""" + BOOT_FAIL = "boot_fail" CORRUPT_DOCKER = "corrupt_docker" CORRUPT_REPOSITORY = "corrupt_repository" CORRUPT_FILESYSTEM = "corrupt_filesystem" @@ -103,6 +104,7 @@ class SuggestionType(StrEnum): ADOPT_DATA_DISK = "adopt_data_disk" CLEAR_FULL_BACKUP = "clear_full_backup" CREATE_FULL_BACKUP = "create_full_backup" + DISABLE_BOOT = "disable_boot" EXECUTE_INTEGRITY = "execute_integrity" EXECUTE_REBOOT = "execute_reboot" EXECUTE_REBUILD = "execute_rebuild" @@ -110,6 +112,7 @@ class SuggestionType(StrEnum): EXECUTE_REMOVE = "execute_remove" EXECUTE_REPAIR = "execute_repair" EXECUTE_RESET = "execute_reset" + EXECUTE_START = "execute_start" EXECUTE_STOP = "execute_stop" EXECUTE_UPDATE = "execute_update" REGISTRY_LOGIN = "registry_login" diff --git a/supervisor/resolution/fixups/addon_disable_boot.py b/supervisor/resolution/fixups/addon_disable_boot.py new file mode 100644 index 00000000000..b3d8052b9dd --- /dev/null +++ b/supervisor/resolution/fixups/addon_disable_boot.py @@ -0,0 +1,48 @@ +"""Helpers to fix addon by disabling boot.""" + +import logging + +from ...const import AddonBoot +from ...coresys import CoreSys +from ..const import ContextType, IssueType, SuggestionType +from .base import FixupBase + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +def setup(coresys: CoreSys) -> FixupBase: + """Check setup function.""" + return FixupAddonDisableBoot(coresys) + + +class FixupAddonDisableBoot(FixupBase): + """Storage class for fixup.""" + + async def process_fixup(self, reference: str | None = None) -> None: + """Initialize the fixup class.""" + if not (addon := self.sys_addons.get(reference, local_only=True)): + _LOGGER.info("Cannot change addon %s as it does not exist", reference) + return + + # Disable boot on addon + addon.boot = AddonBoot.MANUAL + + @property + def suggestion(self) -> SuggestionType: + """Return a SuggestionType enum.""" + return SuggestionType.DISABLE_BOOT + + @property + def context(self) -> ContextType: + """Return a ContextType enum.""" + return ContextType.ADDON + + @property + def issues(self) -> list[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.BOOT_FAIL] + + @property + def auto(self) -> bool: + """Return if a fixup can be apply as auto fix.""" + return False diff --git a/supervisor/resolution/fixups/addon_execute_start.py b/supervisor/resolution/fixups/addon_execute_start.py new file mode 100644 index 00000000000..4eb52612846 --- /dev/null +++ b/supervisor/resolution/fixups/addon_execute_start.py @@ -0,0 +1,59 @@ +"""Helpers to fix addon by starting it.""" + +import logging + +from ...const import AddonState +from ...coresys import CoreSys +from ...exceptions import AddonsError, ResolutionFixupError +from ..const import ContextType, IssueType, SuggestionType +from .base import FixupBase + +_LOGGER: logging.Logger = logging.getLogger(__name__) + + +def setup(coresys: CoreSys) -> FixupBase: + """Check setup function.""" + return FixupAddonExecuteStart(coresys) + + +class FixupAddonExecuteStart(FixupBase): + """Storage class for fixup.""" + + async def process_fixup(self, reference: str | None = None) -> None: + """Initialize the fixup class.""" + if not (addon := self.sys_addons.get(reference, local_only=True)): + _LOGGER.info("Cannot start addon %s as it does not exist", reference) + return + + # Start addon + try: + start_task = await addon.start() + except AddonsError as err: + _LOGGER.error("Could not start %s due to %s", reference, err) + raise ResolutionFixupError() from None + + # Wait for addon start. If it ends up in error or unknown state it's not fixed + await start_task + if addon.state in {AddonState.ERROR, AddonState.UNKNOWN}: + _LOGGER.error("Addon %s could not start successfully", reference) + raise ResolutionFixupError() + + @property + def suggestion(self) -> SuggestionType: + """Return a SuggestionType enum.""" + return SuggestionType.EXECUTE_START + + @property + def context(self) -> ContextType: + """Return a ContextType enum.""" + return ContextType.ADDON + + @property + def issues(self) -> list[IssueType]: + """Return a IssueType enum list.""" + return [IssueType.BOOT_FAIL] + + @property + def auto(self) -> bool: + """Return if a fixup can be apply as auto fix.""" + return False diff --git a/tests/addons/test_addon.py b/tests/addons/test_addon.py index 7ee520a61d4..4f6d21dfa5d 100644 --- a/tests/addons/test_addon.py +++ b/tests/addons/test_addon.py @@ -14,7 +14,7 @@ from supervisor.addons.addon import Addon from supervisor.addons.const import AddonBackupMode from supervisor.addons.model import AddonModel -from supervisor.const import AddonState, BusEvent +from supervisor.const import AddonBoot, AddonState, BusEvent from supervisor.coresys import CoreSys from supervisor.docker.addon import DockerAddon from supervisor.docker.const import ContainerState @@ -24,6 +24,8 @@ from supervisor.store.repository import Repository from supervisor.utils.dt import utcnow +from .test_manager import BOOT_FAIL_ISSUE, BOOT_FAIL_SUGGESTIONS + from tests.common import get_fixture_path from tests.const import TEST_ADDON_SLUG @@ -895,3 +897,32 @@ async def test_addon_manual_only_boot(coresys: CoreSys, install_addon_example: A # However boot mode can change on update and user may have set auto before, ensure it is ignored install_addon_example.boot = "auto" assert install_addon_example.boot == "manual" + + +async def test_addon_start_dismisses_boot_fail( + coresys: CoreSys, install_addon_ssh: Addon +): + """Test a successful start dismisses the boot fail issue.""" + install_addon_ssh.state = AddonState.ERROR + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, [suggestion.type for suggestion in BOOT_FAIL_SUGGESTIONS] + ) + + install_addon_ssh.state = AddonState.STARTED + assert coresys.resolution.issues == [] + assert coresys.resolution.suggestions == [] + + +async def test_addon_disable_boot_dismisses_boot_fail( + coresys: CoreSys, install_addon_ssh: Addon +): + """Test a disabling boot dismisses the boot fail issue.""" + install_addon_ssh.boot = AddonBoot.AUTO + install_addon_ssh.state = AddonState.ERROR + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, [suggestion.type for suggestion in BOOT_FAIL_SUGGESTIONS] + ) + + install_addon_ssh.boot = AddonBoot.MANUAL + assert coresys.resolution.issues == [] + assert coresys.resolution.suggestions == [] diff --git a/tests/addons/test_manager.py b/tests/addons/test_manager.py index e4761037a09..eceadc50f81 100644 --- a/tests/addons/test_manager.py +++ b/tests/addons/test_manager.py @@ -1,6 +1,7 @@ """Test addon manager.""" import asyncio +from collections.abc import AsyncGenerator, Generator from copy import deepcopy from pathlib import Path from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch @@ -25,6 +26,8 @@ DockerNotFound, ) from supervisor.plugins.dns import PluginDns +from supervisor.resolution.const import ContextType, IssueType, SuggestionType +from supervisor.resolution.data import Issue, Suggestion from supervisor.store.addon import AddonStore from supervisor.store.repository import Repository from supervisor.utils import check_exception_chain @@ -33,9 +36,21 @@ from tests.common import load_json_fixture from tests.const import TEST_ADDON_SLUG +BOOT_FAIL_ISSUE = Issue( + IssueType.BOOT_FAIL, ContextType.ADDON, reference=TEST_ADDON_SLUG +) +BOOT_FAIL_SUGGESTIONS = [ + Suggestion( + SuggestionType.EXECUTE_START, ContextType.ADDON, reference=TEST_ADDON_SLUG + ), + Suggestion( + SuggestionType.DISABLE_BOOT, ContextType.ADDON, reference=TEST_ADDON_SLUG + ), +] + @pytest.fixture(autouse=True) -async def fixture_mock_arch_disk() -> None: +async def fixture_mock_arch_disk() -> AsyncGenerator[None]: """Mock supported arch and disk space.""" with ( patch("shutil.disk_usage", return_value=(42, 42, 2 * (1024.0**3))), @@ -45,13 +60,15 @@ async def fixture_mock_arch_disk() -> None: @pytest.fixture(autouse=True) -async def fixture_remove_wait_boot(coresys: CoreSys) -> None: +async def fixture_remove_wait_boot(coresys: CoreSys) -> AsyncGenerator[None]: """Remove default wait boot time for tests.""" coresys.config.wait_boot = 0 @pytest.fixture(name="install_addon_example_image") -def fixture_install_addon_example_image(coresys: CoreSys, repository) -> Addon: +def fixture_install_addon_example_image( + coresys: CoreSys, repository +) -> Generator[Addon]: """Install local_example add-on with image.""" store = coresys.addons.store["local_example_image"] coresys.addons.data.install(store) @@ -114,14 +131,17 @@ async def test_addon_boot_system_error( ): """Test system errors during addon boot.""" install_addon_ssh.boot = AddonBoot.AUTO + assert coresys.resolution.issues == [] + assert coresys.resolution.suggestions == [] with ( patch.object(Addon, "write_options"), patch.object(DockerAddon, "run", side_effect=err), ): await coresys.addons.boot(AddonStartup.APPLICATION) - assert install_addon_ssh.boot == AddonBoot.MANUAL capture_exception.assert_not_called() + assert coresys.resolution.issues == [BOOT_FAIL_ISSUE] + assert coresys.resolution.suggestions == BOOT_FAIL_SUGGESTIONS async def test_addon_boot_user_error( @@ -132,8 +152,9 @@ async def test_addon_boot_user_error( with patch.object(Addon, "write_options", side_effect=AddonConfigurationError): await coresys.addons.boot(AddonStartup.APPLICATION) - assert install_addon_ssh.boot == AddonBoot.MANUAL capture_exception.assert_not_called() + assert coresys.resolution.issues == [BOOT_FAIL_ISSUE] + assert coresys.resolution.suggestions == BOOT_FAIL_SUGGESTIONS async def test_addon_boot_other_error( @@ -148,8 +169,9 @@ async def test_addon_boot_other_error( ): await coresys.addons.boot(AddonStartup.APPLICATION) - assert install_addon_ssh.boot == AddonBoot.AUTO capture_exception.assert_called_once_with(err) + assert coresys.resolution.issues == [BOOT_FAIL_ISSUE] + assert coresys.resolution.suggestions == BOOT_FAIL_SUGGESTIONS async def test_addon_shutdown_error( diff --git a/tests/resolution/fixup/test_addon_disable_boot.py b/tests/resolution/fixup/test_addon_disable_boot.py new file mode 100644 index 00000000000..e89cc9068c1 --- /dev/null +++ b/tests/resolution/fixup/test_addon_disable_boot.py @@ -0,0 +1,40 @@ +"""Test fixup addon disable boot.""" + +from supervisor.addons.addon import Addon +from supervisor.const import AddonBoot +from supervisor.coresys import CoreSys +from supervisor.resolution.const import SuggestionType +from supervisor.resolution.fixups.addon_disable_boot import FixupAddonDisableBoot + +from tests.addons.test_manager import BOOT_FAIL_ISSUE + + +async def test_fixup(coresys: CoreSys, install_addon_ssh: Addon): + """Test fixup disables boot.""" + install_addon_ssh.boot = AddonBoot.AUTO + addon_disable_boot = FixupAddonDisableBoot(coresys) + assert addon_disable_boot.auto is False + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.DISABLE_BOOT], + ) + await addon_disable_boot() + + assert install_addon_ssh.boot == AddonBoot.MANUAL + assert not coresys.resolution.issues + assert not coresys.resolution.suggestions + + +async def test_fixup_no_addon(coresys: CoreSys): + """Test fixup dismisses if addon is missing.""" + addon_disable_boot = FixupAddonDisableBoot(coresys) + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.DISABLE_BOOT], + ) + await addon_disable_boot() + + assert not coresys.resolution.issues + assert not coresys.resolution.suggestions diff --git a/tests/resolution/fixup/test_addon_execute_repair.py b/tests/resolution/fixup/test_addon_execute_repair.py index d5edfca898d..26fe5413054 100644 --- a/tests/resolution/fixup/test_addon_execute_repair.py +++ b/tests/resolution/fixup/test_addon_execute_repair.py @@ -1,4 +1,4 @@ -"""Test fixup core execute repair.""" +"""Test fixup addon execute repair.""" from unittest.mock import MagicMock, patch diff --git a/tests/resolution/fixup/test_addon_execute_start.py b/tests/resolution/fixup/test_addon_execute_start.py new file mode 100644 index 00000000000..26c876cd959 --- /dev/null +++ b/tests/resolution/fixup/test_addon_execute_start.py @@ -0,0 +1,117 @@ +"""Test fixup addon execute start.""" + +from unittest.mock import patch + +import pytest + +from supervisor.addons.addon import Addon +from supervisor.const import AddonState +from supervisor.coresys import CoreSys +from supervisor.docker.addon import DockerAddon +from supervisor.exceptions import DockerError +from supervisor.resolution.const import ContextType, SuggestionType +from supervisor.resolution.data import Suggestion +from supervisor.resolution.fixups.addon_execute_start import FixupAddonExecuteStart + +from tests.addons.test_manager import BOOT_FAIL_ISSUE + +EXECUTE_START_SUGGESTION = Suggestion( + SuggestionType.EXECUTE_START, ContextType.ADDON, reference="local_ssh" +) + + +@pytest.mark.parametrize( + "state", [AddonState.STARTED, AddonState.STARTUP, AddonState.STOPPED] +) +@pytest.mark.usefixtures("path_extern") +async def test_fixup(coresys: CoreSys, install_addon_ssh: Addon, state: AddonState): + """Test fixup starts addon.""" + install_addon_ssh.state = AddonState.UNKNOWN + addon_execute_start = FixupAddonExecuteStart(coresys) + assert addon_execute_start.auto is False + + async def mock_start(*args, **kwargs): + install_addon_ssh.state = state + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.EXECUTE_START], + ) + with ( + patch.object(DockerAddon, "run") as run, + patch.object(Addon, "_wait_for_startup", new=mock_start), + patch.object(Addon, "write_options"), + ): + await addon_execute_start() + run.assert_called_once() + + assert not coresys.resolution.issues + assert not coresys.resolution.suggestions + + +@pytest.mark.usefixtures("path_extern") +async def test_fixup_start_error(coresys: CoreSys, install_addon_ssh: Addon): + """Test fixup fails on start addon failure.""" + install_addon_ssh.state = AddonState.UNKNOWN + addon_execute_start = FixupAddonExecuteStart(coresys) + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.EXECUTE_START], + ) + with ( + patch.object(DockerAddon, "run", side_effect=DockerError) as run, + patch.object(Addon, "write_options"), + ): + await addon_execute_start() + run.assert_called_once() + + assert BOOT_FAIL_ISSUE in coresys.resolution.issues + assert EXECUTE_START_SUGGESTION in coresys.resolution.suggestions + + +@pytest.mark.parametrize("state", [AddonState.ERROR, AddonState.UNKNOWN]) +@pytest.mark.usefixtures("path_extern") +async def test_fixup_wait_start_failure( + coresys: CoreSys, install_addon_ssh: Addon, state: AddonState +): + """Test fixup fails if addon does not complete startup.""" + install_addon_ssh.state = AddonState.UNKNOWN + addon_execute_start = FixupAddonExecuteStart(coresys) + + async def mock_start(*args, **kwargs): + install_addon_ssh.state = state + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.EXECUTE_START], + ) + with ( + patch.object(DockerAddon, "run") as run, + patch.object(Addon, "_wait_for_startup", new=mock_start), + patch.object(Addon, "write_options"), + ): + await addon_execute_start() + run.assert_called_once() + + assert BOOT_FAIL_ISSUE in coresys.resolution.issues + assert EXECUTE_START_SUGGESTION in coresys.resolution.suggestions + + +async def test_fixup_no_addon(coresys: CoreSys): + """Test fixup dismisses if addon is missing.""" + addon_execute_start = FixupAddonExecuteStart(coresys) + + coresys.resolution.add_issue( + BOOT_FAIL_ISSUE, + suggestions=[SuggestionType.EXECUTE_START], + ) + with ( + patch.object(DockerAddon, "run") as run, + patch.object(Addon, "write_options"), + ): + await addon_execute_start() + run.assert_not_called() + + assert not coresys.resolution.issues + assert not coresys.resolution.suggestions