Fixes #740: Expand patterns in settings (#741)

* Fix #740: expand resource settings

* Add quicksuggest to demo config

* Fix bucket in local setting

config/local.ini

@@ -124,6 +124,9 @@ kinto.signer.main-workspace.nimbus-web-preview.to_review_enabled = false
 # crash-reports-ondemand has multi-signoff disabled. See RRA ticket (SA-137)
 kinto.signer.main-workspace.crash-reports-ondemand.to_review_enabled = false
 
+# quicksuggest collections don't new review
+kinto.signer.main-workspace.quicksuggest-(\w+)-(desktop|mobile).to_review_enabled = false
+
 #
 # Simple daemon (see `run.sh start`)
 #

kinto-remote-settings/src/kinto_remote_settings/signer/__init__.py

@@ -3,17 +3,22 @@
 import re
 import sys
 
+import transaction
 from kinto.core import metrics as core_metrics
 from kinto.core import utils as core_utils
+from kinto.core.events import ACTIONS, ResourceChanged
 from pyramid.authorization import Authenticated
-from pyramid.events import ApplicationCreated
-from pyramid.settings import aslist
+from pyramid.events import ApplicationCreated, NewRequest
+from pyramid.settings import asbool, aslist
 
 from .. import __version__
+from . import listeners, utils
+from .backends import heartbeat
 from .events import ReviewApproved, ReviewRejected, ReviewRequested
-from .utils import storage_create_raw
 
 
+IS_RUNNING_MIGRATE = "migrate" in sys.argv
+
 DEFAULT_SETTINGS = {
     "allow_floats": False,
     "auto_create_resources": False,
@@ -37,22 +42,10 @@ def on_review_approved(event):
         metrics_service.count(f"plugins.signer.approved_changes.{bid}.{cid}", count)
 
 
-def includeme(config):
-    # We import stuff here, so that kinto-signer can be installed with `--no-deps`
-    # and used without having this Pyramid ecosystem installed.
-    import transaction
-    from kinto.core.events import ACTIONS, ResourceChanged
-    from pyramid.events import NewRequest
-    from pyramid.settings import asbool
-
-    from . import listeners, utils
-    from .backends import heartbeat
-
-    # Register heartbeat to check signer integration.
-    config.registry.heartbeats["signer"] = heartbeat
+def load_signed_resources_configuration(config):
+    settings = config.get_settings()
 
     # Load settings from KINTO_SIGNER_* environment variables.
-    settings = config.get_settings()
     for setting, default_value in DEFAULT_SETTINGS.items():
         settings[f"signer.{setting}"] = utils.get_first_matching_setting(
             setting_name=setting,
@@ -61,6 +54,14 @@ def includeme(config):
             default=default_value,
         )
 
+    # Expand glob settings into concrete settings using existing objects in DB.
+    # (eg. "signer.main-workspace.magic-(\w+)" -> "signer.main-workspace.magic-word")
+    expanded_settings = utils.expand_collections_glob_settings(
+        config.registry.storage, settings
+    )
+    config.add_settings(expanded_settings)
+    settings.update(**expanded_settings)
+
     # Check source and destination resources are configured.
     resources = utils.parse_resources(settings["signer.resources"])
 
@@ -69,7 +70,7 @@ def includeme(config):
     # For example, consider the case where resource is ``/buckets/dev -> /buckets/prod``
     # and there is a setting ``signer.dev.recipes.signer_backend = foo``
     output_resources = resources.copy()
-    for key, resource in resources.items():
+    for resource in resources.values():
         # If collection is not None, there is nothing to expand :)
         if resource["source"]["collection"] is not None:
             continue
@@ -154,9 +155,11 @@ def includeme(config):
             r, ["source", "destination", "preview", "to_review_enabled"]
         )
         for r in resources.values()
+        if "(" not in str(r["source"])  # do not show patterns
     ]
     message = "Digital signatures for integrity and authenticity of records."
     docs = "https://github.com/Kinto/kinto-signer#kinto-signer"
+    config.registry.api_capabilities.pop("signer", None)
     config.add_api_capability(
         "signer",
         message,
@@ -168,6 +171,32 @@ def includeme(config):
         **global_settings,
     )
 
+    return resources
+
+
+def includeme(config):
+    # Register heartbeat to check signer integration.
+    config.registry.heartbeats["signer"] = heartbeat
+
+    resources = load_signed_resources_configuration(config)
+
+    settings = config.get_settings()
+
+    global_settings = {
+        k: v
+        for k, v in config.registry.api_capabilities["signer"].items()
+        if k in ("editors_group", "reviewers_group", "to_review_enabled")
+    }
+
+    # Since we have settings that can contain glob patterns, we refresh the settings
+    # and exposed resources when a new collection is created.
+    config.add_subscriber(
+        lambda _: load_signed_resources_configuration(config),
+        ResourceChanged,
+        for_actions=(ACTIONS.CREATE,),
+        for_resources=("collection",),
+    )
+
     config.add_subscriber(on_review_approved, ReviewApproved)
 
     config.add_subscriber(
@@ -276,7 +305,7 @@ def auto_create_resources(event, resources):
             collection = resource["source"]["collection"]
 
             bucket_uri = f"/buckets/{bucket}"
-            storage_create_raw(
+            utils.storage_create_raw(
                 storage_backend=storage,
                 permission_backend=permission,
                 resource_name="bucket",
@@ -289,7 +318,7 @@ def auto_create_resources(event, resources):
             # If resource is configured for specific collection, create it too.
             if collection:
                 collection_uri = f"{bucket_uri}/collections/{collection}"
-                storage_create_raw(
+                utils.storage_create_raw(
                     storage_backend=storage,
                     permission_backend=permission,
                     resource_name="collection",
@@ -302,7 +331,7 @@ def auto_create_resources(event, resources):
     # Create resources on startup (except when executing `migrate`).
     if (
         asbool(settings.get("signer.auto_create_resources", False))
-        and "migrate" not in sys.argv
+        and not IS_RUNNING_MIGRATE
     ):
         config.add_subscriber(
             functools.partial(

kinto-remote-settings/src/kinto_remote_settings/signer/utils.py

@@ -1,4 +1,5 @@
 import logging
+import re
 import ssl
 from collections import OrderedDict
 from enum import Enum
@@ -293,3 +294,53 @@ def fetch_cert(url):
         cert_pem.encode("utf8"), backend=crypto_default_backend()
     )
     return cert
+
+
+def expand_collections_glob_settings(
+    storage, settings: dict[str, any]
+) -> dict[str, any]:
+    r"""
+    Expand glob patterns in settings using actual bucket and collection names from storage.
+
+    Example:
+        "kinto.signer.main-workspace.quicksuggest-(\w+).to_review_enabled"
+        expands to:
+        "kinto.signer.main-workspace.quicksuggest-fr.to_review_enabled"
+        "kinto.signer.main-workspace.quicksuggest-en.to_review_enabled"
+    """
+    if (
+        hasattr(storage, "get_installed_version")
+        and storage.get_installed_version() is None
+    ):
+        # The DB is a memory backend, or is not ready yet, do not even try to list collections.
+        return settings
+
+    # Fetch all buckets
+    buckets = storage.list_all(parent_id="", resource_name="bucket")
+
+    # Fetch all collections for each bucket
+    collections = [
+        (bucket["id"], collection["id"])
+        for bucket in buckets
+        for collection in storage.list_all(
+            parent_id=f"/buckets/{bucket['id']}", resource_name="collection"
+        )
+    ]
+
+    expanded_settings = {}
+
+    for key, value in settings.items():
+        tokens = key.split(".")
+        # Skip if not a supported glob pattern (for simplicity, just for to_review_enabled now)
+        if "(" not in key or len(tokens) != 4 or tokens[-1] != "to_review_enabled":
+            expanded_settings[key] = value
+            continue
+
+        prefix, bucket_pattern, collection_pattern, setting = tokens
+
+        # Match and expand glob patterns
+        for bid, cid in collections:
+            if re.match(bucket_pattern, bid) and re.match(collection_pattern, cid):
+                expanded_settings[f"{prefix}.{bid}.{cid}.{setting}"] = value
+
+    return expanded_settings

kinto-remote-settings/tests/signer/test_plugin_setup.py

@@ -731,3 +731,33 @@ def test_related_objects_are_all_deleted(self):
         self.app.get(
             "/buckets/stage/groups/a-reviewers", headers=self.headers, status=404
         )
+
+
+class ExpandedSettingsTest(BaseWebTest, PatchAutographMixin, unittest.TestCase):
+    @classmethod
+    def get_app_settings(cls, extras=None):
+        settings = super(cls, ExpandedSettingsTest).get_app_settings(extras)
+        settings["signer.to_review_enabled"] = "true"
+        settings["signer.main-workspace.magic-(\\w+).to_review_enabled"] = "false"
+        return settings
+
+    def setUp(self):
+        super().setUp()
+        self.app.put_json("/buckets/main-workspace", headers=self.headers)
+
+    def test_expanded_settings_are_updated_on_collection_creation(self):
+        server_info = self.app.get("/").json
+        resources = server_info["capabilities"]["signer"]["resources"]
+        source_collections = {entry["source"]["collection"] for entry in resources}
+        assert "magic-word" not in source_collections
+        assert "magic-(\\w+)" not in source_collections
+
+        self.app.put_json(
+            "/buckets/main-workspace/collections/magic-word", headers=self.headers
+        )
+
+        server_info = self.app.get("/").json
+        resources = server_info["capabilities"]["signer"]["resources"]
+        source_collections = {entry["source"]["collection"] for entry in resources}
+        assert "magic-word" in source_collections
+        assert "magic-(\\w+)" not in source_collections

kinto-remote-settings/tests/signer/test_utils.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 
 import pytest
 from kinto_remote_settings.signer import utils
@@ -227,3 +228,38 @@ def test_cannot_repeat_resources(self):
         """
         with pytest.raises(ConfigurationError):
             utils.parse_resources(raw_resources)
+
+
+def test_expand_collections_glob_settings():
+    settings = {
+        "signer.some_setting": "foo",
+        "signer.some-bucket.to_review_enabled": False,
+        "signer.main-workspace.intermediates.to_review_enabled": True,
+        "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).to_review_enabled": True,
+        "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).some_other": 42,
+    }
+
+    storage = mock.MagicMock()
+    storage.list_all.side_effect = (
+        [{"id": "security-state"}, {"id": "main-workspace"}, {"id": "main"}],
+        [
+            {"id": "intermediates"},
+        ],
+        [
+            {"id": "quicksuggest-fr-mobile"},
+            {"id": "quicksuggest-en-desktop"},
+            {"id": "quicksuggest-fr-tablet"},
+        ],
+        [],
+    )
+
+    expanded_settings = utils.expand_collections_glob_settings(storage, settings)
+
+    assert expanded_settings == {
+        "signer.main-workspace.quicksuggest-fr-mobile.to_review_enabled": True,
+        "signer.main-workspace.quicksuggest-en-desktop.to_review_enabled": True,
+        "signer.main-workspace.quicksuggest-(\\w+)-(desktop|mobile).some_other": 42,
+        "signer.main-workspace.intermediates.to_review_enabled": True,
+        "signer.some-bucket.to_review_enabled": False,
+        "signer.some_setting": "foo",
+    }