diff --git a/config/local.ini b/config/local.ini index 9a03c418..4b5ca4bb 100644 --- a/config/local.ini +++ b/config/local.ini @@ -124,6 +124,9 @@ kinto.signer.main-workspace.nimbus-web-preview.to_review_enabled = false # crash-reports-ondemand has multi-signoff disabled. See RRA ticket (SA-137) kinto.signer.main-workspace.crash-reports-ondemand.to_review_enabled = false +# quicksuggest collections don't new review +kinto.signer.main-workspace.quicksuggest-(\w+)-(desktop|mobile).to_review_enabled = false + # # Simple daemon (see `run.sh start`) # diff --git a/kinto-remote-settings/src/kinto_remote_settings/signer/__init__.py b/kinto-remote-settings/src/kinto_remote_settings/signer/__init__.py index c30c3762..e2112f63 100644 --- a/kinto-remote-settings/src/kinto_remote_settings/signer/__init__.py +++ b/kinto-remote-settings/src/kinto_remote_settings/signer/__init__.py @@ -3,17 +3,22 @@ import re import sys +import transaction from kinto.core import metrics as core_metrics from kinto.core import utils as core_utils +from kinto.core.events import ACTIONS, ResourceChanged from pyramid.authorization import Authenticated -from pyramid.events import ApplicationCreated -from pyramid.settings import aslist +from pyramid.events import ApplicationCreated, NewRequest +from pyramid.settings import asbool, aslist from .. import __version__ +from . import listeners, utils +from .backends import heartbeat from .events import ReviewApproved, ReviewRejected, ReviewRequested -from .utils import storage_create_raw +IS_RUNNING_MIGRATE = "migrate" in sys.argv + DEFAULT_SETTINGS = { "allow_floats": False, "auto_create_resources": False, @@ -37,22 +42,10 @@ def on_review_approved(event): metrics_service.count(f"plugins.signer.approved_changes.{bid}.{cid}", count) -def includeme(config): - # We import stuff here, so that kinto-signer can be installed with `--no-deps` - # and used without having this Pyramid ecosystem installed. - import transaction - from kinto.core.events import ACTIONS, ResourceChanged - from pyramid.events import NewRequest - from pyramid.settings import asbool - - from . import listeners, utils - from .backends import heartbeat - - # Register heartbeat to check signer integration. - config.registry.heartbeats["signer"] = heartbeat +def load_signed_resources_configuration(config): + settings = config.get_settings() # Load settings from KINTO_SIGNER_* environment variables. - settings = config.get_settings() for setting, default_value in DEFAULT_SETTINGS.items(): settings[f"signer.{setting}"] = utils.get_first_matching_setting( setting_name=setting, @@ -61,6 +54,14 @@ def includeme(config): default=default_value, ) + # Expand glob settings into concrete settings using existing objects in DB. + # (eg. "signer.main-workspace.magic-(\w+)" -> "signer.main-workspace.magic-word") + expanded_settings = utils.expand_collections_glob_settings( + config.registry.storage, settings + ) + config.add_settings(expanded_settings) + settings.update(**expanded_settings) + # Check source and destination resources are configured. resources = utils.parse_resources(settings["signer.resources"]) @@ -69,7 +70,7 @@ def includeme(config): # For example, consider the case where resource is ``/buckets/dev -> /buckets/prod`` # and there is a setting ``signer.dev.recipes.signer_backend = foo`` output_resources = resources.copy() - for key, resource in resources.items(): + for resource in resources.values(): # If collection is not None, there is nothing to expand :) if resource["source"]["collection"] is not None: continue @@ -154,9 +155,11 @@ def includeme(config): r, ["source", "destination", "preview", "to_review_enabled"] ) for r in resources.values() + if "(" not in str(r["source"]) # do not show patterns ] message = "Digital signatures for integrity and authenticity of records." docs = "https://github.com/Kinto/kinto-signer#kinto-signer" + config.registry.api_capabilities.pop("signer", None) config.add_api_capability( "signer", message, @@ -168,6 +171,32 @@ def includeme(config): **global_settings, ) + return resources + + +def includeme(config): + # Register heartbeat to check signer integration. + config.registry.heartbeats["signer"] = heartbeat + + resources = load_signed_resources_configuration(config) + + settings = config.get_settings() + + global_settings = { + k: v + for k, v in config.registry.api_capabilities["signer"].items() + if k in ("editors_group", "reviewers_group", "to_review_enabled") + } + + # Since we have settings that can contain glob patterns, we refresh the settings + # and exposed resources when a new collection is created. + config.add_subscriber( + lambda _: load_signed_resources_configuration(config), + ResourceChanged, + for_actions=(ACTIONS.CREATE,), + for_resources=("collection",), + ) + config.add_subscriber(on_review_approved, ReviewApproved) config.add_subscriber( @@ -276,7 +305,7 @@ def auto_create_resources(event, resources): collection = resource["source"]["collection"] bucket_uri = f"/buckets/{bucket}" - storage_create_raw( + utils.storage_create_raw( storage_backend=storage, permission_backend=permission, resource_name="bucket", @@ -289,7 +318,7 @@ def auto_create_resources(event, resources): # If resource is configured for specific collection, create it too. if collection: collection_uri = f"{bucket_uri}/collections/{collection}" - storage_create_raw( + utils.storage_create_raw( storage_backend=storage, permission_backend=permission, resource_name="collection", @@ -302,7 +331,7 @@ def auto_create_resources(event, resources): # Create resources on startup (except when executing `migrate`). if ( asbool(settings.get("signer.auto_create_resources", False)) - and "migrate" not in sys.argv + and not IS_RUNNING_MIGRATE ): config.add_subscriber( functools.partial( diff --git a/kinto-remote-settings/src/kinto_remote_settings/signer/utils.py b/kinto-remote-settings/src/kinto_remote_settings/signer/utils.py index 7ea5b859..6d65835e 100644 --- a/kinto-remote-settings/src/kinto_remote_settings/signer/utils.py +++ b/kinto-remote-settings/src/kinto_remote_settings/signer/utils.py @@ -1,4 +1,5 @@ import logging +import re import ssl from collections import OrderedDict from enum import Enum @@ -293,3 +294,53 @@ def fetch_cert(url): cert_pem.encode("utf8"), backend=crypto_default_backend() ) return cert + + +def expand_collections_glob_settings( + storage, settings: dict[str, any] +) -> dict[str, any]: + r""" + Expand glob patterns in settings using actual bucket and collection names from storage. + + Example: + "kinto.signer.main-workspace.quicksuggest-(\w+).to_review_enabled" + expands to: + "kinto.signer.main-workspace.quicksuggest-fr.to_review_enabled" + "kinto.signer.main-workspace.quicksuggest-en.to_review_enabled" + """ + if ( + hasattr(storage, "get_installed_version") + and storage.get_installed_version() is None + ): + # The DB is a memory backend, or is not ready yet, do not even try to list collections. + return settings + + # Fetch all buckets + buckets = storage.list_all(parent_id="", resource_name="bucket") + + # Fetch all collections for each bucket + collections = [ + (bucket["id"], collection["id"]) + for bucket in buckets + for collection in storage.list_all( + parent_id=f"/buckets/{bucket['id']}", resource_name="collection" + ) + ] + + expanded_settings = {} + + for key, value in settings.items(): + tokens = key.split(".") + # Skip if not a supported glob pattern (for simplicity, just for to_review_enabled now) + if "(" not in key or len(tokens) != 4 or tokens[-1] != "to_review_enabled": + expanded_settings[key] = value + continue + + prefix, bucket_pattern, collection_pattern, setting = tokens + + # Match and expand glob patterns + for bid, cid in collections: + if re.match(bucket_pattern, bid) and re.match(collection_pattern, cid): + expanded_settings[f"{prefix}.{bid}.{cid}.{setting}"] = value + + return expanded_settings diff --git a/kinto-remote-settings/tests/signer/test_plugin_setup.py b/kinto-remote-settings/tests/signer/test_plugin_setup.py index 75b1d416..46667451 100644 --- a/kinto-remote-settings/tests/signer/test_plugin_setup.py +++ b/kinto-remote-settings/tests/signer/test_plugin_setup.py @@ -731,3 +731,33 @@ def test_related_objects_are_all_deleted(self): self.app.get( "/buckets/stage/groups/a-reviewers", headers=self.headers, status=404 ) + + +class ExpandedSettingsTest(BaseWebTest, PatchAutographMixin, unittest.TestCase): + @classmethod + def get_app_settings(cls, extras=None): + settings = super(cls, ExpandedSettingsTest).get_app_settings(extras) + settings["signer.to_review_enabled"] = "true" + settings["signer.main-workspace.magic-(\\w+).to_review_enabled"] = "false" + return settings + + def setUp(self): + super().setUp() + self.app.put_json("/buckets/main-workspace", headers=self.headers) + + def test_expanded_settings_are_updated_on_collection_creation(self): + server_info = self.app.get("/").json + resources = server_info["capabilities"]["signer"]["resources"] + source_collections = {entry["source"]["collection"] for entry in resources} + assert "magic-word" not in source_collections + assert "magic-(\\w+)" not in source_collections + + self.app.put_json( + "/buckets/main-workspace/collections/magic-word", headers=self.headers + ) + + server_info = self.app.get("/").json + resources = server_info["capabilities"]["signer"]["resources"] + source_collections = {entry["source"]["collection"] for entry in resources} + assert "magic-word" in source_collections + assert "magic-(\\w+)" not in source_collections diff --git a/kinto-remote-settings/tests/signer/test_utils.py b/kinto-remote-settings/tests/signer/test_utils.py index 11c765d6..2065eb37 100644 --- a/kinto-remote-settings/tests/signer/test_utils.py +++ b/kinto-remote-settings/tests/signer/test_utils.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock import pytest from kinto_remote_settings.signer import utils @@ -227,3 +228,38 @@ def test_cannot_repeat_resources(self): """ with pytest.raises(ConfigurationError): utils.parse_resources(raw_resources) + + +def test_expand_collections_glob_settings(): + settings = { + "signer.some_setting": "foo", + "signer.some-bucket.to_review_enabled": False, + "signer.main-workspace.intermediates.to_review_enabled": True, + "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).to_review_enabled": True, + "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).some_other": 42, + } + + storage = mock.MagicMock() + storage.list_all.side_effect = ( + [{"id": "security-state"}, {"id": "main-workspace"}, {"id": "main"}], + [ + {"id": "intermediates"}, + ], + [ + {"id": "quicksuggest-fr-mobile"}, + {"id": "quicksuggest-en-desktop"}, + {"id": "quicksuggest-fr-tablet"}, + ], + [], + ) + + expanded_settings = utils.expand_collections_glob_settings(storage, settings) + + assert expanded_settings == { + "signer.main-workspace.quicksuggest-fr-mobile.to_review_enabled": True, + "signer.main-workspace.quicksuggest-en-desktop.to_review_enabled": True, + "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).some_other": 42, + "signer.main-workspace.intermediates.to_review_enabled": True, + "signer.some-bucket.to_review_enabled": False, + "signer.some_setting": "foo", + }