From b741a1bba95f31514b27d8cc7a2dfedfdfff014b Mon Sep 17 00:00:00 2001 From: jeanluc Date: Tue, 14 May 2024 22:57:46 +0200 Subject: [PATCH] Add beacon module to monitor and renew cached leases --- .copier-answers.yml | 1 + changelog/53.added.md | 1 + docs/index.rst | 1 + docs/ref/beacons/index.rst | 12 + .../saltext.vault.beacons.vault_lease.rst | 5 + src/saltext/vault/beacons/__init__.py | 0 src/saltext/vault/beacons/vault_lease.py | 253 ++++++++++ src/saltext/vault/states/vault_db.py | 9 +- tests/conftest.py | 14 +- tests/functional/beacons/__init__.py | 0 tests/functional/beacons/conftest.py | 40 ++ tests/functional/beacons/test_vault_lease.py | 438 ++++++++++++++++++ tests/integration/conftest.py | 4 +- .../files/vault/policies/salt_minion.hcl | 5 + tests/integration/states/test_vault_db.py | 253 ++++++++++ tests/unit/beacons/__init__.py | 0 tests/unit/beacons/test_vault_lease.py | 257 ++++++++++ 17 files changed, 1281 insertions(+), 12 deletions(-) create mode 100644 changelog/53.added.md create mode 100644 docs/ref/beacons/index.rst create mode 100644 docs/ref/beacons/saltext.vault.beacons.vault_lease.rst create mode 100644 src/saltext/vault/beacons/__init__.py create mode 100644 src/saltext/vault/beacons/vault_lease.py create mode 100644 tests/functional/beacons/__init__.py create mode 100644 tests/functional/beacons/conftest.py create mode 100644 tests/functional/beacons/test_vault_lease.py create mode 100644 tests/integration/states/test_vault_db.py create mode 100644 tests/unit/beacons/__init__.py create mode 100644 tests/unit/beacons/test_vault_lease.py diff --git a/.copier-answers.yml b/.copier-answers.yml index 0a19135e..b52bb178 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -7,6 +7,7 @@ author_email: saltproject@vmware.com docs_url: https://salt-extensions.github.io/saltext-vault/ license: apache loaders: + - beacon - module - pillar - runner diff --git a/changelog/53.added.md b/changelog/53.added.md new file mode 100644 index 00000000..ae76669e --- /dev/null +++ b/changelog/53.added.md @@ -0,0 +1 @@ +Added `vault_lease` beacon module to monitor and renew cached leases diff --git a/docs/index.rst b/docs/index.rst index 259dcccf..eb5f1f6a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,6 +46,7 @@ Found a bug or missing a feature? :caption: Provided Modules :hidden: + ref/beacons/index ref/modules/index ref/pillar/index ref/runners/index diff --git a/docs/ref/beacons/index.rst b/docs/ref/beacons/index.rst new file mode 100644 index 00000000..f65b6c03 --- /dev/null +++ b/docs/ref/beacons/index.rst @@ -0,0 +1,12 @@ +.. all-saltext.vault.beacons: + +______________ +Beacon Modules +______________ + +.. currentmodule:: saltext.vault.beacons + +.. autosummary:: + :toctree: + + vault_lease diff --git a/docs/ref/beacons/saltext.vault.beacons.vault_lease.rst b/docs/ref/beacons/saltext.vault.beacons.vault_lease.rst new file mode 100644 index 00000000..c7f7de67 --- /dev/null +++ b/docs/ref/beacons/saltext.vault.beacons.vault_lease.rst @@ -0,0 +1,5 @@ +``vault_lease`` +=============== + +.. automodule:: saltext.vault.beacons.vault_lease + :members: diff --git a/src/saltext/vault/beacons/__init__.py b/src/saltext/vault/beacons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/saltext/vault/beacons/vault_lease.py b/src/saltext/vault/beacons/vault_lease.py new file mode 100644 index 00000000..5f38bc57 --- /dev/null +++ b/src/saltext/vault/beacons/vault_lease.py @@ -0,0 +1,253 @@ +""" +Beacon for the Vault integration. Sends events when a +lease's TTL undercuts a specified value. By default, also +tries to renew leases before sending an event. + +.. versionadded:: 1.1.0 + +Event description +----------------- + +When a lease undercuts its minimum TTL, an event is sent. + +The event tag's format is: ``salt/beacon//vault_lease_/expire`` + +The event data contains (non-exhaustive): + +* ``expires_in`` - number of seconds left until the lease is revoked by Vault (can be ``-1`` if already revoked) +* ``lease_id`` - the lease ID of the expiring lease +* ``ckey`` - the cache key of the expiring lease +* ``meta`` - custom metadata, e.g. for use in a reactor +* ``expired`` - if the lease is already expired + +Example configuration +--------------------- +.. code-block:: yaml + + beacons: + vault_basic: + - beacon_module: vault_lease + - leases: + - db.database.dynamic.basic_lease.default + + vault_advanced: + - beacon_module: vault_lease + - leases: + db.database.dynamic.write_stuff.default: {} + db.database.dynamic.monitoring.default: + renew: false + db.database.dynamic.read_stuff.default: + min_ttl: 6h + meta: + sls: read.stuff + - min_ttl: 1h + - meta: + sls: write.stuff + - check_server: true + +.. _beacon-state-example: + +Example for enabling beacon via state +------------------------------------- +This beacon can be added dynamically when explicitly caching +database leases. + +.. code-block:: yaml + + Important Vault lease is cached: + vault_db.creds_cached: + - name: my_important_role + - valid_for: 6h # minimum TTL for the lease to be returned by get_creds + - revoke_delay: 30m + - beacon: true # also add a beacon for monitoring + - beacon_interval: 300 # interval between beacon runs + - min_ttl: 12h # minimum TTL for the beacon to accept the lease as valid + - meta: my.important.state # can be used with a reactor + - order: first # leases should be cached early + +Configuration reference +----------------------- +.. vconf:: lease_beacon.leases + +``leases`` + The leases to monitor, referenced by their cache keys. + This can be a string (single lease), list (multiple leases) + or mapping (multiple leases with parameter overrides). + +.. vconf:: lease_beacon.min_ttl + +``min_ttl`` + The minimum TTL a monitored lease should have. + Can be overridden per configured lease in :vconf:`lease_beacon.leases`. + If a ``min_ttl`` was set on the lease during its creation, + this value must be equal or greater to have any effect. + Defaults to ``300``. + +.. vconf:: lease_beacon.check_server + +``check_server`` + Whether cached leases should be validated with the Vault server + before declaring them as valid. + Can be overridden per configured lease in :vconf:`lease_beacon.leases`. + There is no equivalent parameter that can be set on the lease during + its creation currently. + Defaults to false. + +.. vconf:: lease_beacon.meta + +``meta`` + Arbitrary metadata to include in expiry events. + Can be overridden per configured lease in :vconf:`lease_beacon.leases`. + If ``meta`` was set on the lease during creation, the corresponding + value takes precedence. If both values are either mappings or lists, + they will be merged together. + +.. vconf:: lease_beacon.renew + +``renew`` + Before sending an event, try to renew the lease as needed. + Defaults to true. +""" + +import logging + +import salt.utils.beacons +import salt.utils.dictupdate as dup + +import saltext.vault.utils.vault as vault +from saltext.vault.utils.vault.helpers import timestring_map + +log = logging.getLogger(__name__) + + +__virtualname__ = "vault_lease" + + +def __virtual__(): + return __virtualname__ + + +def validate(config): + """ + Validate the beacon configuration + """ + if not isinstance(config, list): + return False, "Configuration for vault_lease must be a list" + config = salt.utils.beacons.list_to_dict(config) + if "leases" not in config: + return False, "Requires monitored lease(s) cache key(s) in `leases`" + if not isinstance(config["leases"], (dict, list, str)): + return False, "`leases` must be a dict, list or str" + + if isinstance(config["leases"], str): + if "*" in config["leases"]: + return False, "`leases` does not support globs" + else: + if any("*" in lease for lease in config["leases"]): + return False, "`leases` does not support globs" + if isinstance(config["leases"], dict) and any( + not isinstance(cfg, dict) for cfg in config["leases"].values() + ): + return False, "`leases` mapping values must be dicts" + + return True, "Valid beacon configuration." + + +def beacon(config): + """ + Watch the configured lease(s). + """ + config = _render_config(config) + # background processes should not pass __context__ + store = vault.get_lease_store(__opts__, {}) + events = [] + for lease, lease_config in config["leases"].items(): + info = store.list_info(match=lease) + if not info: + events.append(_enrich_info(lease, lease_config, {"expires_in": -1, "expired": True})) + continue + lease_info = info[lease] + effective_config = _merge_lease_config(lease_config, lease_info) + if effective_config.get("check_server"): + try: + store.lookup(lease_info["lease_id"]) + except vault.VaultNotFoundError: + store.revoke(lease_info["lease_id"], delta=lease_info.get("revoke_delay")) + lease_info["expires_in"] = -1 + lease_info["expired"] = True + events.append(_enrich_info(lease, effective_config, lease_info)) + continue + if lease_info["expired"]: + events.append(_enrich_info(lease, effective_config, lease_info)) + continue + if timestring_map(effective_config["min_ttl"]) >= lease_info["expires_in"]: + if not effective_config.get("renew", True): + events.append(_enrich_info(lease, effective_config, lease_info)) + continue + # attempt renewal + res = store.get( + lease, + valid_for=effective_config["min_ttl"], + revoke=False, + check_server=effective_config.get("check_server", False), + ) + if not res: + events.append(_enrich_info(lease, effective_config, lease_info)) + continue + return events + + +def _enrich_info(lease, effective_config, info): + info["ckey"] = lease + info["meta"] = effective_config.get("meta") + info["min_ttl"] = effective_config.get("min_ttl", 300) + info["check_server"] = effective_config.get("check_server") + info.pop("id", None) + info["tag"] = "expire" + return info + + +def _render_config(cfg): + config = salt.utils.beacons.list_to_dict(cfg) + if isinstance(config["leases"], str): + config["leases"] = {config["leases"]: {}} + if not isinstance(config["leases"], dict): + config["leases"] = {lease: {} for lease in config["leases"]} + defaults = {} + for param in ("min_ttl", "meta", "check_server", "renew"): + if param in config: + defaults[param] = config[param] + return { + "leases": { + lease: {**defaults, **lease_config} for lease, lease_config in config["leases"].items() + } + } + + +def _merge_lease_config(cfg, lease): + if cfg.get("min_ttl") is not None and lease.get("min_ttl") is not None: + cfg["min_ttl"] = ( + lease["min_ttl"] + if timestring_map(lease["min_ttl"]) >= timestring_map(cfg["min_ttl"]) + else cfg["min_ttl"] + ) + elif lease.get("min_ttl") is not None: + cfg["min_ttl"] = lease["min_ttl"] + elif "min_ttl" not in cfg: + cfg["min_ttl"] = 300 + cfg["meta"] = _merge_meta(cfg.get("meta"), lease.get("meta")) + return cfg + + +def _merge_meta(default, ovrr): + if ovrr is None: + return default + default = default if default is not None else {} + for val in (default, ovrr): + if not isinstance(val, (dict, list)): + return ovrr + if type(default) is not type(ovrr): + return ovrr + if isinstance(default, list): + return default + ovrr + return dup.merge(default, ovrr, merge_lists=True) or None diff --git a/src/saltext/vault/states/vault_db.py b/src/saltext/vault/states/vault_db.py index 0a4d89b8..976bb9dd 100644 --- a/src/saltext/vault/states/vault_db.py +++ b/src/saltext/vault/states/vault_db.py @@ -588,6 +588,8 @@ def creds_cached( This function is mosly intended to associate a specific credential with a beacon that warns about expiry and allows to run an associated state to reconfigure an application with new credentials. + See the :py:mod:`vault_lease beacon module ` + for an :ref:`example state to configure a lease together with a beacon `. name The name of the database role. @@ -721,9 +723,10 @@ def creds_uncached( .. note:: - This function is mosly intended to associate a specific credential with - a beacon that warns about expiry and allows to run an associated state to - reconfigure an application with new credentials. + This function is mosly intended to remove a cached lease and its + beacon. See :py:func:`creds_cached` for a more detailed description. + To remove the associated beacon together with the lease, just pass + ``beacon: true`` as a parameter to this state. name The name of the database role. diff --git a/tests/conftest.py b/tests/conftest.py index b7eb6b38..3a5968ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def salt_factories_config(): } -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def master_config_defaults(vault_port): """ This default configuration ensures the master issues authentication @@ -67,7 +67,7 @@ def master_config_defaults(vault_port): "issue": { "token": { "params": { - "uses": 0, + "num_uses": 0, } } }, @@ -83,7 +83,7 @@ def master_config_defaults(vault_port): } -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def master_config_overrides(): """ You can override the default configuration per package by overriding this @@ -92,14 +92,14 @@ def master_config_overrides(): return {} -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def master(salt_factories, master_config_defaults, master_config_overrides): return salt_factories.salt_master_daemon( random_string("master-"), defaults=master_config_defaults, overrides=master_config_overrides ) -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def minion_config_defaults(vault_port): """ The default minion configuration ensures that the minion works in --local @@ -123,7 +123,7 @@ def minion_config_defaults(vault_port): } -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def minion_config_overrides(): """ You can override the default configuration per package by overriding this @@ -132,7 +132,7 @@ def minion_config_overrides(): return {} -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def minion(master, minion_config_defaults, minion_config_overrides): return master.salt_minion_daemon( random_string("minion-"), defaults=minion_config_defaults, overrides=minion_config_overrides diff --git a/tests/functional/beacons/__init__.py b/tests/functional/beacons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/functional/beacons/conftest.py b/tests/functional/beacons/conftest.py new file mode 100644 index 00000000..df798998 --- /dev/null +++ b/tests/functional/beacons/conftest.py @@ -0,0 +1,40 @@ +import pytest + + +@pytest.fixture +def beacons(loaders): + def _beacons(self): + """ + The beacons loaded by the salt loader. + We need to patch this in since it's currently not + supported by pytest-salt-factories. + """ + # Do not move these deferred imports. It allows running against a Salt + # onedir build in salt's repo checkout. + import salt.loader # pylint: disable=import-outside-toplevel + + if self._beacons is None: + self._beacons = salt.loader.beacons( + self.opts, + functions=self.modules, + context=self.context, + loaded_base_name=self.loaded_base_name, + ) + return self._beacons + + def _reload_all(self): + loaders.reload_all() + if self._beacons is not None: + self._beacons.clean_modules() + self._beacons.clear() + self._beacons = None + + try: + loaders._beacons = None + type(loaders).beacons = property(_beacons) + loaders.reload_all = _reload_all + yield loaders.beacons + finally: + loaders.reset_state() + delattr(loaders, "_beacons") + delattr(type(loaders), "beacons") diff --git a/tests/functional/beacons/test_vault_lease.py b/tests/functional/beacons/test_vault_lease.py new file mode 100644 index 00000000..df62039b --- /dev/null +++ b/tests/functional/beacons/test_vault_lease.py @@ -0,0 +1,438 @@ +import pytest +from saltfactories.utils import random_string + +from tests.support.mysql import MySQLImage +from tests.support.mysql import create_mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_container # pylint: disable=unused-import +from tests.support.vault import vault_delete +from tests.support.vault import vault_disable_secret_engine +from tests.support.vault import vault_enable_secret_engine +from tests.support.vault import vault_list +from tests.support.vault import vault_revoke +from tests.support.vault import vault_write + +pytest.importorskip("docker") + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("vault", "getent"), + pytest.mark.usefixtures("vault_container_version"), + pytest.mark.parametrize("vault_container_version", ("latest",), indirect=True), +] + + +@pytest.fixture(scope="module") +def minion_config_overrides(vault_port): + return { + "vault": { + "auth": { + "method": "token", + "token": "testsecret", + }, + "cache": { + "backend": "disk", # ensure a persistent cache is available for get_creds + }, + "server": { + "url": f"http://127.0.0.1:{vault_port}", + }, + } + } + + +@pytest.fixture(scope="module") +def mysql_image(): + version = "10.3" + return MySQLImage( + name="mariadb", + tag=version, + container_id=random_string(f"mariadb-{version}-"), + ) + + +@pytest.fixture +def role_args_common(): + return { + "db_name": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + + +@pytest.fixture +def testrole(): + return { + "default_ttl": 3600, + "max_ttl": 7200, + } + + +@pytest.fixture +def testdb(mysql_container, container_host_ref): + return { + "plugin_name": "mysql-database-plugin", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp({container_host_ref}:{mysql_container.mysql_port})/", + "allowed_roles": "testrole,teststaticrole,testreissuerole", + "username": "root", + "password": mysql_container.mysql_passwd, + } + + +@pytest.fixture(scope="module", autouse=True) +def db_engine(vault_container_version): # pylint: disable=unused-argument + assert vault_enable_secret_engine("database") + yield + assert vault_disable_secret_engine("database") + + +@pytest.fixture +def connection_setup(testdb): + try: + vault_write("database/config/testdb", **testdb) + assert "testdb" in vault_list("database/config") + yield + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + + +@pytest.fixture(params=[["testrole"]]) +def roles_setup(connection_setup, request, role_args_common): # pylint: disable=unused-argument + try: + for role_name in request.param: + role_args = request.getfixturevalue(role_name) + role_args.update(role_args_common) + vault_write(f"database/roles/{role_name}", **role_args) + assert role_name in vault_list("database/roles") + yield + finally: + for role_name in request.param: + if role_name in vault_list("database/roles"): + vault_delete(f"database/roles/{role_name}") + assert role_name not in vault_list("database/roles") + + +@pytest.fixture +def vault_db(modules): + try: + yield modules.vault_db + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + if "testrole" in vault_list("database/roles"): + vault_delete("database/roles/testrole") + assert "testrole" not in vault_list("database/roles") + + +@pytest.fixture(params=({},)) +def lease_creation_params(request): + defaults = {"name": "testrole"} + defaults.update(request.param) + return defaults + + +@pytest.fixture(params=(False,)) +def _multi_lease(request): + return request.param + + +@pytest.fixture(params=({},)) +def beacon_config(request, _multi_lease, existing_lease): + req = request.param.copy() + datatype = req.pop("leases_type", str if not _multi_lease else list) + if _multi_lease: + alt_lease = request.getfixturevalue("existing_alt_lease") + data = {} + if datatype is str: + data["leases"] = existing_lease + elif datatype is list: + data["leases"] = [existing_lease] + if _multi_lease: + data["leases"].append(alt_lease) + else: + data["leases"] = {existing_lease: req.pop("per_lease_params", {})} + if _multi_lease: + data["leases"][alt_lease] = req.pop("per_lease_alt_params", {}) + data.update(req) + return [{k: v} for k, v in data.items()] + + +@pytest.fixture +def existing_lease( + roles_setup, lease_creation_params, vault_db, loaders +): # pylint: disable=unused-argument + ckey = ".".join( + [ + "db", + lease_creation_params.get("mount", "database"), + "dynamic", + lease_creation_params["name"], + lease_creation_params.get("cache", "default"), + ] + ) + lease = vault_db.get_creds(**lease_creation_params) + assert lease + # We need to clear the context because in the test suite, the beacon modules + # are running in a different one than the execution modules and the lease + # has already been cached in the context of the execution module. + # This means it does not pick up changes to the cached files, but we need + # it to check changes in the tests. + loaders.context.clear() + return ckey # revocation is handled in vault_db + + +@pytest.fixture(params=({"cache": "alt"},)) +def existing_alt_lease( + request, roles_setup, lease_creation_params, vault_db, loaders +): # pylint: disable=unused-argument + params = request.param + ckey = ".".join( + [ + "db", + params.get("mount", lease_creation_params.get("mount", "database")), + "dynamic", + params.get("name", lease_creation_params["name"]), + params.get("cache", lease_creation_params.get("cache", "default")), + ] + ) + lease = vault_db.get_creds(**lease_creation_params, **params) + assert lease + # We need to clear the context because in the test suite, the beacon modules + # are running in a different one than the execution modules and the lease + # has already been cached in the context of the execution module. + # This means it does not pick up changes to the cached files, but we need + # it to check changes in the tests. + loaders.context.clear() + return ckey # revocation is handled in vault_db + + +@pytest.fixture +def revoked_lease(existing_lease, vault_db): + lease_id = vault_db.list_cached()[existing_lease]["lease_id"] + assert vault_revoke(lease_id) + return existing_lease + + +@pytest.fixture +def beacon(beacons): + yield beacons.vault_lease.beacon + + +@pytest.mark.usefixtures("existing_alt_lease") +@pytest.mark.usefixtures("_multi_lease") +@pytest.mark.parametrize("_multi_lease", (False, True), indirect=True) +@pytest.mark.parametrize( + "beacon_config", ({"check_server": False}, {"check_server": True}), indirect=True +) +def test_beacon_valid(beacon, beacon_config): + ret = beacon(beacon_config) + assert ret == [] + + +@pytest.mark.parametrize("beacon_config", ({"leases": "foo.bar.baz"},), indirect=True) +def test_beacon_missing(beacon, beacon_config): + ret = beacon(beacon_config) + assert ret == [ + { + "check_server": None, + "ckey": "foo.bar.baz", + "expired": True, + "expires_in": -1, + "meta": None, + "min_ttl": 300, + "tag": "expire", + } + ] + + +@pytest.mark.parametrize("beacon_config", ({"leases": ["foo.bar", "foo.baz"]},), indirect=True) +def test_beacon_missing_multi(beacon, beacon_config): + ret = beacon(beacon_config) + assert ret == [ + { + "check_server": None, + "ckey": "foo.bar", + "expired": True, + "expires_in": -1, + "meta": None, + "min_ttl": 300, + "tag": "expire", + }, + { + "check_server": None, + "ckey": "foo.baz", + "expired": True, + "expires_in": -1, + "meta": None, + "min_ttl": 300, + "tag": "expire", + }, + ] + + +@pytest.mark.usefixtures("existing_alt_lease") +@pytest.mark.usefixtures("_multi_lease") +@pytest.mark.parametrize("_multi_lease", (True,), indirect=True) +@pytest.mark.usefixtures("revoked_lease") +def test_beacon_revoked_not_check_server(beacon, beacon_config): + ret = beacon(beacon_config) + assert ret == [] + + +@pytest.mark.usefixtures("existing_alt_lease") +@pytest.mark.usefixtures("_multi_lease") +@pytest.mark.parametrize("_multi_lease", (True,), indirect=True) +@pytest.mark.usefixtures("revoked_lease", "lease_creation_params") +@pytest.mark.parametrize( + "beacon_config,lease_creation_params", + ( + ({"check_server": True}, {}), + ({"leases_type": dict, "per_lease_params": {"check_server": True}}, {}), + ), + indirect=True, +) +def test_beacon_revoked_check_server(beacon, beacon_config): + ret = beacon(beacon_config) + assert len(ret) == 1 + ret = ret[0] + _assert_evt(ret, check_server=True, expired=True, expires_in=-1) + + +@pytest.mark.usefixtures("beacon_config", "lease_creation_params") +@pytest.mark.parametrize( + "beacon_config,lease_creation_params", + ( + ({"min_ttl": 7000}, {}), + ({"leases_type": dict, "per_lease_params": {"min_ttl": 7000}}, {}), + ({}, {"valid_for": 7000}), + ({"min_ttl": 300}, {"valid_for": 7000}), + ({"leases_type": dict, "per_lease_params": {"min_ttl": 300}}, {"valid_for": 7000}), + ), + indirect=True, +) +def test_beacon_min_ttl(beacon, beacon_config, vault_db, existing_lease): + ret = beacon(beacon_config) + assert ret == [] + info = vault_db.list_cached()[existing_lease] + assert info["duration"] == 7000 + + +@pytest.mark.usefixtures("beacon_config", "existing_lease") +@pytest.mark.parametrize( + "beacon_config", + ({"renew": False, "min_ttl": 7000},), + indirect=True, +) +def test_beacon_not_renew(beacon, beacon_config): + ret = beacon(beacon_config) + assert len(ret) == 1 + ret = ret[0] + _assert_evt(ret, min_ttl=7000) + + +@pytest.mark.usefixtures("beacon_config", "lease_creation_params") +@pytest.mark.parametrize( + "beacon_config,lease_creation_params", + ( + ({"min_ttl": 8000}, {}), + ({"leases_type": dict, "per_lease_params": {"min_ttl": 8000}}, {}), + ({}, {"valid_for": 8000}), + ({"min_ttl": 300}, {"valid_for": 8000}), + ({"leases_type": dict, "per_lease_params": {"min_ttl": 300}}, {"valid_for": 8000}), + ), + indirect=True, +) +def test_beacon_min_ttl_unattainable(beacon, beacon_config): + ret = beacon(beacon_config) + assert len(ret) == 1 + ret = ret[0] + _assert_evt(ret, min_ttl=8000) + assert ret["expires_in"] > 3590 + + +@pytest.mark.usefixtures("beacon_config", "lease_creation_params") +@pytest.mark.parametrize( + "beacon_config,lease_creation_params,expected_meta", + ( + ({"meta": "foo.bar"}, {}, "foo.bar"), + ({"leases_type": dict, "per_lease_params": {"meta": "foo.bar"}}, {}, "foo.bar"), + ({}, {"meta": "foo.bar"}, "foo.bar"), + ({"meta": "foo.bar"}, {"meta": "foo.baz"}, "foo.baz"), + ( + {"leases_type": dict, "per_lease_params": {"meta": "foo.bar"}}, + {"meta": "foo.baz"}, + "foo.baz", + ), + ({"meta": "foo.bar"}, {"meta": ["foo.baz"]}, ["foo.baz"]), + ({"meta": ["foo.bar"]}, {"meta": "foo.baz"}, "foo.baz"), + ({"meta": ["foo.bar"]}, {"meta": {"foo": "baz"}}, {"foo": "baz"}), + ({"meta": ["foo.bar"]}, {"meta": ["foo.baz"]}, ["foo.bar", "foo.baz"]), + ( + {"meta": {"foo": {"bar": True}}}, + {"meta": {"foo": {"bar": False}}}, + {"foo": {"bar": False}}, + ), + ( + {"meta": {"foo": {"bar": True}}}, + {"meta": {"foo": {"baz": False}}}, + {"foo": {"bar": True, "baz": False}}, + ), + ( + {"meta": {"foo": {"bar": [True]}}}, + {"meta": {"foo": {"bar": [False]}}}, + {"foo": {"bar": [True, False]}}, + ), + ({"meta": "foo"}, {"meta": {"foo": {"bar": False}}}, {"foo": {"bar": False}}), + ), + indirect=("beacon_config", "lease_creation_params"), +) +def test_beacon_meta(beacon, beacon_config, expected_meta): + beacon_config.append({"min_ttl": 10000}) + ret = beacon(beacon_config) + assert len(ret) == 1 + ret = ret[0] + _assert_evt(ret, min_ttl=10000, meta=expected_meta) + + +def _assert_evt(evt, *remove, **expected): + assert set(evt) == { + "meta", + "creation_time", + "duration", + "expired", + "revoke_delay", + "tag", + "renew_increment", + "renewable", + "min_ttl", + "check_server", + "lease_id", + "ckey", + "expire_time", + "expires_in", + } + expected = { + "check_server": None, + "ckey": "db.database.dynamic.testrole.default", + # For renewals, we can't know what the max_ttl is, so this will be the default + # duration. + "duration": 3600, + "expired": False, + "meta": None, + "min_ttl": 300, + "renew_increment": None, + "renewable": True, + "revoke_delay": None, + "tag": "expire", + **expected, + } + expected.update(expected) + for unwanted in remove: + expected.pop(unwanted, None) + for param, val in expected.items(): + assert evt[param] == val diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9bd94587..86cd8d78 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,13 +1,13 @@ import pytest -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def master(master): with master.started(): yield master -@pytest.fixture(scope="package") +@pytest.fixture(scope="module") def minion(minion): with minion.started(): yield minion diff --git a/tests/integration/files/vault/policies/salt_minion.hcl b/tests/integration/files/vault/policies/salt_minion.hcl index 9759c6f9..1c52b5aa 100644 --- a/tests/integration/files/vault/policies/salt_minion.hcl +++ b/tests/integration/files/vault/policies/salt_minion.hcl @@ -27,3 +27,8 @@ path "sys/policy" { path "sys/policy/*" { capabilities = ["read", "create", "update", "delete"] } + +# Request database credentials in integration test +path "database/creds/*" { + capabilities = ["read"] +} diff --git a/tests/integration/states/test_vault_db.py b/tests/integration/states/test_vault_db.py new file mode 100644 index 00000000..95ffa910 --- /dev/null +++ b/tests/integration/states/test_vault_db.py @@ -0,0 +1,253 @@ +from textwrap import dedent + +import pytest +import salt.utils.beacons +from saltfactories.utils import random_string + +from tests.support.mysql import MySQLImage +from tests.support.mysql import create_mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_combo # pylint: disable=unused-import +from tests.support.mysql import mysql_container # pylint: disable=unused-import +from tests.support.vault import vault_delete +from tests.support.vault import vault_disable_secret_engine +from tests.support.vault import vault_enable_secret_engine +from tests.support.vault import vault_list +from tests.support.vault import vault_revoke +from tests.support.vault import vault_write + +pytest.importorskip("docker") + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("vault", "getent"), + pytest.mark.usefixtures("vault_container_version"), + pytest.mark.parametrize("vault_container_version", ("latest",), indirect=True), +] + + +@pytest.fixture(scope="module") +def master_config_overrides(): + return { + "vault": { + "cache": { + "backend": "disk", + }, + }, + } + + +@pytest.fixture(scope="module") +def mysql_image(): + version = "10.3" + return MySQLImage( + name="mariadb", + tag=version, + container_id=random_string(f"mariadb-{version}-"), + ) + + +@pytest.fixture +def role_args_common(): + return { + "db_name": "testdb", + "creation_statements": r"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT ON *.* TO '{{name}}'@'%';", + } + + +@pytest.fixture +def testrole(): + return { + "default_ttl": 3600, + "max_ttl": 86400, + } + + +@pytest.fixture +def testreissuerole(): + return { + "default_ttl": 180, + "max_ttl": 180, + } + + +@pytest.fixture +def teststaticrole(mysql_container): + return { + "db_name": "testdb", + "rotation_period": 86400, + "username": mysql_container.mysql_user, + } + + +@pytest.fixture +def testdb(mysql_container, container_host_ref): + return { + "plugin_name": "mysql-database-plugin", + "connection_url": f"{{{{username}}}}:{{{{password}}}}@tcp({container_host_ref}:{mysql_container.mysql_port})/", + "allowed_roles": "testrole,teststaticrole,testreissuerole", + "username": "root", + "password": mysql_container.mysql_passwd, + } + + +@pytest.fixture(scope="module", autouse=True) +def db_engine(vault_container_version): # pylint: disable=unused-argument + assert vault_enable_secret_engine("database") + yield + assert vault_disable_secret_engine("database") + + +@pytest.fixture +def connection_setup(testdb): + try: + vault_write("database/config/testdb", **testdb) + assert "testdb" in vault_list("database/config") + yield + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + + +@pytest.fixture(params=[["testrole"]], autouse=True) +def roles_setup(connection_setup, request, role_args_common): # pylint: disable=unused-argument + try: + for role_name in request.param: + role_args = request.getfixturevalue(role_name) + role_args.update(role_args_common) + vault_write(f"database/roles/{role_name}", **role_args) + assert role_name in vault_list("database/roles") + yield + finally: + for role_name in request.param: + if role_name in vault_list("database/roles"): + vault_delete(f"database/roles/{role_name}") + assert role_name not in vault_list("database/roles") + + +@pytest.fixture(autouse=True) +def _cleanup(): + try: + yield + finally: + # prevent dangling leases, which prevent disabling the secret engine + assert vault_revoke("database/creds", prefix=True) + if "testdb" in vault_list("database/config"): + vault_delete("database/config/testdb") + assert "testdb" not in vault_list("database/config") + if "testrole" in vault_list("database/roles"): + vault_delete("database/roles/testrole") + assert "testrole" not in vault_list("database/roles") + if "teststaticrole" in vault_list("database/static-roles"): + vault_delete("database/static-roles/teststaticrole") + assert "teststaticrole" not in vault_list("database/static-roles") + + +@pytest.fixture +def _lease_beacon(master, salt_call_cli): + state_contents = dedent( + """ + Vault lease is cached: + vault_db.creds_cached: + - name: testrole + # When the lease is requested, it is valid for + # at least this amount of time + - valid_for: 900 + - beacon: true + - beacon_interval: 300 + - check_server: true + # The beacon ensures this min_ttl, must be >= valid_for if set + - min_ttl: 1200 + - meta: other.state.or.something.else + """ + ) + states = master.state_tree.base.temp_file("vault_lease.sls", state_contents) + try: + with states: + yield "vault_lease" + finally: + beacon = "vault_lease_db.database.dynamic.testrole.default" + salt_call_cli.run("beacons.delete", beacon) + res = salt_call_cli.run("beacons.list", return_yaml=False) + assert res.returncode == 0 + assert beacon not in res.data + + +@pytest.fixture +def _lease_beacon_absent(master, salt_call_cli): + state_contents = dedent( + """ + Vault lease is not cached: + vault_db.creds_uncached: + - name: testrole + - beacon: true + """ + ) + states = master.state_tree.base.temp_file("vault_lease_absent.sls", state_contents) + try: + with states: + yield "vault_lease_absent" + finally: + beacon = "vault_lease_db.database.dynamic.testrole.default" + salt_call_cli.run("beacons.delete", beacon) + res = salt_call_cli.run("beacons.list", return_yaml=False) + assert res.returncode == 0 + assert beacon not in res.data + + +def test_creds_cached_mod_beacon(salt_call_cli, _lease_beacon, _lease_beacon_absent): + """ + Ensure beacons can be added as part of caching a lease. + """ + ret = salt_call_cli.run("state.apply", _lease_beacon) + assert ret.returncode == 0 + + # Ensure the lease is actually cached + ckey = "db.database.dynamic.testrole.default" + ret = salt_call_cli.run("vault_db.list_cached") + assert ret.returncode == 0 + assert ret.data + assert ckey in ret.data + + # Ensure the beacon has been created with the correct config + # The beacon modules have a really weird API. + # Dynamic beacons are added as an "opts" item. + # We need to explicitly disable YAML output to get the Python object. + # Each beacon config param is its own single-item (dict) list. + # Also, most beacon functions are prone to crash on even the most basic + # unexpected circumstances. Oh well. They are nice when they work. + ret = salt_call_cli.run( + "beacons.list", return_yaml=False, include_pillar=False, include_opts=True + ) + assert ret.returncode == 0 + assert ret.data + beacons_config = {} + for beacon, conf in ret.data.items(): + beacons_config[beacon] = salt.utils.beacons.list_to_dict(conf) + beacon = "vault_lease_db.database.dynamic.testrole.default" + assert beacon in beacons_config + assert beacons_config[beacon] == { + "beacon_module": "vault_lease", + "interval": 300, + "leases": ckey, + "min_ttl": 1200, + "meta": "other.state.or.something.else", + "check_server": True, + } + # I wanted to test if the beacon is executed with a high min_ttl + # and low interval, but it seems beacons are not scheduled at all here. + ret = salt_call_cli.run("state.apply", _lease_beacon_absent) + assert ret.returncode == 0 + + # Now remove the lease and its beacon again + ret = salt_call_cli.run("vault_db.list_cached") + assert ret.returncode == 0 + assert ckey not in ret.data + + ret = salt_call_cli.run( + "beacons.list", return_yaml=False, include_pillar=False, include_opts=True + ) + assert ret.returncode == 0 + assert beacon not in ret.data diff --git a/tests/unit/beacons/__init__.py b/tests/unit/beacons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/beacons/test_vault_lease.py b/tests/unit/beacons/test_vault_lease.py new file mode 100644 index 00000000..338c8ff1 --- /dev/null +++ b/tests/unit/beacons/test_vault_lease.py @@ -0,0 +1,257 @@ +import pytest + +import saltext.vault.beacons.vault_lease as lease + + +@pytest.fixture +def configure_loader_modules(): + return { + lease: { + "__grains__": {"id": "test-minion"}, + } + } + + +@pytest.mark.parametrize( + "config,exp", + ( + ({}, "Configuration for vault_lease must be a list"), + ([], "Requires monitored lease(s) cache key(s) in `leases`"), + ([{"leases": 123}], "`leases` must be a dict, list or str"), + ([{"leases": "foo"}], True), + ([{"leases": "foo.*"}], "`leases` does not support globs"), + ([{"leases": ["foo", "bar"]}], True), + ([{"leases": ["foo", "bar.*"]}], "`leases` does not support globs"), + ([{"leases": {"foo": "foo", "bar": "bar"}}], "`leases` mapping values must be dicts"), + ([{"leases": {"foo": {}, "bar": {}}}], True), + ([{"leases": {"foo": {"min_ttl": "1d"}, "bar": {}}}], True), + ([{"leases": {"foo": {"min_ttl": "1d"}, "bar.*": {}}}], "`leases` does not support globs"), + ), +) +def test_validate(config, exp): + res, msg = lease.validate(config) + if exp is True: + assert res is True + else: + assert msg == exp + + +@pytest.mark.parametrize( + "config,exp", + ( + ([{"leases": "foo"}], {"leases": {"foo": {}}}), + ( + [{"leases": "foo"}, {"min_ttl": 42}, {"check_server": True}, {"meta": "foo"}], + {"leases": {"foo": {"min_ttl": 42, "check_server": True, "meta": "foo"}}}, + ), + ([{"leases": ["foo", "bar"]}], {"leases": {"foo": {}, "bar": {}}}), + ( + [{"leases": ["foo", "bar"]}, {"min_ttl": 42}, {"check_server": True}, {"meta": "foo"}], + { + "leases": { + "foo": {"min_ttl": 42, "check_server": True, "meta": "foo"}, + "bar": {"min_ttl": 42, "check_server": True, "meta": "foo"}, + } + }, + ), + ([{"leases": {"foo": {}, "bar": {}}}], {"leases": {"foo": {}, "bar": {}}}), + ( + [ + {"leases": {"foo": {}, "bar": {}}}, + {"min_ttl": 42}, + {"check_server": True}, + {"meta": "foo"}, + ], + { + "leases": { + "foo": {"min_ttl": 42, "check_server": True, "meta": "foo"}, + "bar": {"min_ttl": 42, "check_server": True, "meta": "foo"}, + } + }, + ), + ( + [ + {"leases": {"foo": {"min_ttl": 1337}, "bar": {"check_server": False}}}, + {"min_ttl": 42}, + {"check_server": True}, + {"meta": "foo"}, + ], + { + "leases": { + "foo": {"min_ttl": 1337, "check_server": True, "meta": "foo"}, + "bar": {"min_ttl": 42, "check_server": False, "meta": "foo"}, + } + }, + ), + ( + [ + {"leases": {"foo": {"min_ttl": None}, "bar": {"meta": "bar"}}}, + {"min_ttl": 42}, + {"check_server": True}, + {"meta": "foo"}, + ], + { + "leases": { + "foo": {"min_ttl": None, "check_server": True, "meta": "foo"}, + "bar": {"min_ttl": 42, "check_server": True, "meta": "bar"}, + } + }, + ), + ( + [ + {"leases": {"foo": {"meta": ["foo"]}, "bar": {"meta": {"bar": True}}}}, + {"meta": "foo"}, + ], + {"leases": {"foo": {"meta": ["foo"]}, "bar": {"meta": {"bar": True}}}}, + ), + ( + [ + {"leases": {"foo": {"meta": ["baz"]}, "bar": {"meta": {"bar": True}}}}, + {"meta": ["foo"]}, + ], + {"leases": {"foo": {"meta": ["baz"]}, "bar": {"meta": {"bar": True}}}}, + ), + ( + [ + {"leases": {"foo": {"meta": ["baz"]}, "bar": {"meta": {"bar": True}}}}, + {"meta": {"foo": True}}, + ], + {"leases": {"foo": {"meta": ["baz"]}, "bar": {"meta": {"bar": True}}}}, + ), + ( + [{"leases": ["foo", "bar"]}, {"renew": False}], + {"leases": {"foo": {"renew": False}, "bar": {"renew": False}}}, + ), + ( + [{"leases": {"foo": {"renew": True}, "bar": {}}}, {"renew": False}], + {"leases": {"foo": {"renew": True}, "bar": {"renew": False}}}, + ), + ), +) +def test_render_config(config, exp): + res = lease._render_config(config) + assert res == exp + + +@pytest.mark.parametrize( + "cfg,info,exp", + ( + ({}, {"min_ttl": 1234, "meta": None}, {"min_ttl": 1234, "meta": None}), + ({}, {"min_ttl": "1h", "meta": None}, {"min_ttl": "1h", "meta": None}), + ({"min_ttl": "1h"}, {"min_ttl": None, "meta": None}, {"min_ttl": "1h", "meta": None}), + ( + {"check_server": True}, + {"min_ttl": None, "meta": None}, + {"min_ttl": 300, "meta": None, "check_server": True}, + ), + ( + {"check_server": False}, + {"min_ttl": None, "meta": None}, + {"min_ttl": 300, "meta": None, "check_server": False}, + ), + ({"min_ttl": "2h"}, {"min_ttl": 3600, "meta": None}, {"min_ttl": "2h", "meta": None}), + ({"min_ttl": "1h"}, {"min_ttl": "2h", "meta": None}, {"min_ttl": "2h", "meta": None}), + ({}, {"min_ttl": None, "meta": "foo"}, {"min_ttl": 300, "meta": "foo"}), + ({"min_ttl": 42}, {"min_ttl": None, "meta": None}, {"min_ttl": 42, "meta": None}), + ({"meta": 123}, {"min_ttl": None, "meta": None}, {"min_ttl": 300, "meta": 123}), + ({"meta": ["foo"]}, {"min_ttl": None, "meta": None}, {"min_ttl": 300, "meta": ["foo"]}), + ( + {"meta": {"foo": True}}, + {"min_ttl": None, "meta": None}, + {"min_ttl": 300, "meta": {"foo": True}}, + ), + ({"meta": "foo"}, {"min_ttl": None, "meta": "bar"}, {"min_ttl": 300, "meta": "bar"}), + ({"meta": ["foo"]}, {"min_ttl": None, "meta": "bar"}, {"min_ttl": 300, "meta": "bar"}), + ( + {"meta": {"foo": True}}, + {"min_ttl": None, "meta": "bar"}, + {"min_ttl": 300, "meta": "bar"}, + ), + ( + {"meta": {"foo": True}}, + {"min_ttl": None, "meta": ["bar"]}, + {"min_ttl": 300, "meta": ["bar"]}, + ), + ( + {"meta": ["foo"]}, + {"min_ttl": None, "meta": {"bar": True}}, + {"min_ttl": 300, "meta": {"bar": True}}, + ), + ( + {"meta": ["foo"]}, + {"min_ttl": None, "meta": ["bar"]}, + {"min_ttl": 300, "meta": ["foo", "bar"]}, + ), + ( + {"meta": {"foo": True}}, + {"min_ttl": None, "meta": {"bar": True}}, + {"min_ttl": 300, "meta": {"foo": True, "bar": True}}, + ), + ( + {"meta": {"foo": ["a"]}}, + {"min_ttl": None, "meta": {"foo": ["b"]}}, + {"min_ttl": 300, "meta": {"foo": ["a", "b"]}}, + ), + ( + {"meta": {"foo": True}}, + {"min_ttl": None, "meta": {"foo": False}}, + {"min_ttl": 300, "meta": {"foo": False}}, + ), + ), +) +def test_merge_lease_config(cfg, info, exp): + res = lease._merge_lease_config(cfg, info) + assert res == exp + + +@pytest.mark.parametrize( + "cfg,info,exp", + ( + ( + {"min_ttl": 1234}, + {"min_ttl": 42, "meta": None}, + {"ckey": "test.lease", "min_ttl": 1234, "meta": None, "check_server": None}, + ), + ( + {"min_ttl": 300, "meta": "foo"}, + {"min_ttl": None, "meta": None}, + {"ckey": "test.lease", "min_ttl": 300, "meta": "foo", "check_server": None}, + ), + ( + {"min_ttl": 300, "meta": ["foo"]}, + {"min_ttl": None, "meta": None}, + {"ckey": "test.lease", "min_ttl": 300, "meta": ["foo"], "check_server": None}, + ), + ( + {"min_ttl": 300, "meta": {"foo": True}}, + {"min_ttl": None, "meta": None}, + {"ckey": "test.lease", "min_ttl": 300, "meta": {"foo": True}, "check_server": None}, + ), + ( + {"min_ttl": 300, "check_server": False}, + {"min_ttl": None, "meta": None}, + {"ckey": "test.lease", "min_ttl": 300, "meta": None, "check_server": False}, + ), + ( + {"min_ttl": 300, "check_server": True}, + {"min_ttl": None, "meta": None}, + {"ckey": "test.lease", "min_ttl": 300, "meta": None, "check_server": True}, + ), + ( + {"min_ttl": 300, "check_server": True, "meta": "foo.bar"}, + {"expires_in": -1, "expired": True}, + { + "ckey": "test.lease", + "min_ttl": 300, + "meta": "foo.bar", + "check_server": True, + "expires_in": -1, + "expired": True, + }, + ), + ), +) +def test_enrich_info(cfg, info, exp): + exp["tag"] = "expire" + res = lease._enrich_info("test.lease", cfg, info) + assert res == exp