Add Support for Dynamic Settings (#1837)

Issue: AAH-2009

.compose.env.example

@@ -85,3 +85,6 @@ ENABLE_SIGNING=1
 ## Override the github urls for social auth
 # SOCIAL_AUTH_GITHUB_BASE_URL=
 # SOCIAL_AUTH_GITHUB_API_URL=
+
+# Enable Dynamic Settings
+# PULP_GALAXY_DYNAMIC_SETTINGS=true

CHANGES/2009.feature

@@ -0,0 +1 @@
+Add support for dynamic settings

dev/docker-compose.yml

@@ -122,6 +122,8 @@ services:
 
   redis:
     image: "redis:5"
+    ports:
+      - "6379:6379"
     volumes:
       - "redis_data:/data"
 

docs/config/options.md

@@ -172,6 +172,7 @@ Here is a [diagram explaining](https://www.xmind.net/m/VPSF59/#) the loading ord
 | `GALAXY_SIGNATURE_UPLOAD_ENABLED`  | Used by UI to hide/show the upload buttons for signature, Default: `False` |
 | `GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL`  | Approval dashboard and move endpoint must require signature?, Default: `False` |
 | `GALAXY_MINIMUM_PASSWORD_LENGTH` |  Minimum password lenght for validation, Default: 9 |
+| `GALAXY_DYNAMIC_SETTINGS`  | Enables dynamic settings feature, Default `False` |
 
 For SSO Keycloak configuration see [keycloak](../dev/docker_environment.md#keycloak)
 

galaxy_ng/app/dynaconf_hooks.py

@@ -1,11 +1,17 @@
 import json
+import logging
 import ldap
 import pkg_resources
 import os
 import re
 from typing import Any, Dict, List
 from django_auth_ldap.config import LDAPSearch
 from dynaconf import Dynaconf, Validator
+from galaxy_ng.app.dynamic_settings import DYNAMIC_SETTINGS_SCHEMA
+from django.apps import apps
+
+
+logger = logging.getLogger(__name__)
 
 
 def post(settings: Dynaconf) -> Dict[str, Any]:
@@ -42,6 +48,9 @@ def post(settings: Dynaconf) -> Dict[str, Any]:
     # rest framework auth classes too.
     data.update(configure_authentication_classes(settings, data))
 
+    # This must go last, so that all the default settings are loaded before dynamic and validation
+    data.update(configure_dynamic_settings(settings))
+
     validate(settings)
     return data
 
@@ -582,3 +591,85 @@ def validate(settings: Dynaconf) -> None:
     )
 
     settings.validators.validate()
+
+
+def configure_dynamic_settings(settings: Dynaconf) -> Dict[str, Any]:
+    """Dynaconf 3.2.2 allows registration of hooks on methods `get` and `as_dict`
+
+    For galaxy this enables the Dynamic Settings feature, which triggers a
+    specified function after every key is accessed.
+
+    So after the normal get process, the registered hook will be able to
+    change the value before it is returned allowing reading overrides from
+    database and cache.
+    """
+    if settings.get("GALAXY_DYNAMIC_SETTINGS") is not True:
+        return {}
+
+    # Perform lazy imports here to avoid breaking when system runs with older
+    # dynaconf versions
+    try:
+        from dynaconf.hooking import Hook, Action, HookValue
+        from dynaconf import DynaconfFormatError, DynaconfParseError
+        from dynaconf.loaders.base import SourceMetadata
+        from dynaconf.base import Settings
+    except ImportError as exc:
+        # Graceful degradation for dynaconf < 3.2.3 where  method hooking is not available
+        logger.error(
+            "Galaxy Dynamic Settings requires Dynaconf >=3.2.3, "
+            "system will work normally but dynamic settings from database will be ignored: %s",
+            str(exc)
+        )
+        return {}
+
+    logger.info("Enabling Dynamic Settings Feature")
+
+    def read_settings_from_cache_or_db(
+        temp_settings: Settings,
+        value: HookValue,
+        key: str,
+        *args,
+        **kwargs
+    ) -> Any:
+        """A function to be attached on Dynaconf Afterget hook.
+        Load everything from settings cache or db, process parsing and mergings,
+        returns the desired key value
+        """
+        if not apps.ready or key.upper() not in DYNAMIC_SETTINGS_SCHEMA:
+            # If app is starting up or key is not on allowed list bypass and just return the value
+            return value.value
+
+        # lazy import because it can't happen before apps are ready
+        from galaxy_ng.app.tasks.settings_cache import get_settings_from_cache, get_settings_from_db
+        if data := get_settings_from_cache():
+            metadata = SourceMetadata(loader="hooking", identifier="cache")
+        else:
+            data = get_settings_from_db()
+            if data:
+                metadata = SourceMetadata(loader="hooking", identifier="db")
+
+        # This is the main part, it will update temp_settings with data coming from settings db
+        # and by calling update it will process dynaconf parsing and merging.
+        try:
+            if data:
+                temp_settings.update(data, loader_identifier=metadata, tomlfy=True)
+        except (DynaconfFormatError, DynaconfParseError) as exc:
+            logger.error("Error loading dynamic settings: %s", str(exc))
+
+        if not data:
+            logger.debug("Dynamic settings are empty, reading key %s from default sources", key)
+        elif key in [_k.split("__")[0] for _k in data]:
+            logger.debug("Dynamic setting for key: %s loaded from %s", key, metadata.identifier)
+        else:
+            logger.debug(
+                "Key %s not on db/cache, %s other keys loaded from %s",
+                key, len(data), metadata.identifier
+            )
+
+        return temp_settings.get(key, value.value)
+
+    return {
+        "_registered_hooks": {
+            Action.AFTER_GET: [Hook(read_settings_from_cache_or_db)]
+        }
+    }

galaxy_ng/app/dynamic_settings.py

@@ -0,0 +1,65 @@
+from dynaconf import Validator
+
+DYNAMIC_SETTINGS_SCHEMA = {
+    "GALAXY_REQUIRE_CONTENT_APPROVAL": {
+        "validator": Validator(is_type_of=bool),
+        "schema": {
+            "type": "boolean",
+            "enum": ["true", "false"],
+            "default": False,
+            "description": "Require content approval before it can be published",
+        }
+    },
+    "GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL": {
+        "validator": Validator(is_type_of=bool),
+        "schema": {
+            "type": "boolean",
+            "enum": ["true", "false"],
+            "default": "false",
+            "description": "Require signature for content approval",
+        }
+    },
+    "GALAXY_SIGNATURE_UPLOAD_ENABLED": {
+        "validator": Validator(is_type_of=bool),
+        "schema": {
+            "type": "boolean",
+            "enum": ["true", "false"],
+            "default": "false",
+            "description": "Enable signature upload",
+        }
+    },
+    "GALAXY_AUTO_SIGN_COLLECTIONS": {
+        "validator": Validator(is_type_of=bool),
+        "schema": {
+            "type": "boolean",
+            "enum": ["true", "false"],
+            "default": "false",
+            "description": "Automatically sign collections during approval/upload",
+        }
+    },
+    "GALAXY_FEATURE_FLAGS": {
+        "validator": Validator(is_type_of=dict),
+        "schema": {
+            "type": "object",
+            "properties": {
+                "execution_environments": {
+                    "type": "boolean",
+                    "enum": ["true", "false"],
+                    "default": "false",
+                    "description": "Enable execution environments",
+                },
+            },
+            "default": {},
+            "description": "Feature flags for galaxy_ng",
+        },
+    },
+    # For 1816 PR
+    "INSIGHTS_TRACKING_STATE": {},
+    "AUTOMATION_ANALYTICS_URL": {},
+    "REDHAT_USERNAME": {},
+    "REDHAT_PASSWORD": {},
+    "AUTOMATION_ANALYTICS_LAST_GATHERED": {},
+    "AUTOMATION_ANALYTICS_LAST_ENTRIES": {},
+    "FOO": {},
+    "BAR": {},
+}

galaxy_ng/app/management/commands/galaxy-settings.py

@@ -0,0 +1,134 @@
+from contextlib import suppress
+from django.core.management.base import BaseCommand
+from galaxy_ng.app.models.config import Setting
+from django.conf import settings
+from dynaconf.utils import upperfy
+from galaxy_ng.app.dynamic_settings import DYNAMIC_SETTINGS_SCHEMA
+
+
+class Command(BaseCommand):
+    """This command sets, read, delete Galaxy Setting.
+
+    Examples:
+    django-admin galaxy-settings set --key=foo --value=bar
+    django-admin galaxy-settings set --key=foo --value=bar --is-secret
+
+    django-admin galaxy-settings get --key=foo
+    django-admin galaxy-settings get --key=foo --raw
+
+    django-admin galaxy-settings delete --key=foo --all-versions
+    django-admin galaxy-settings delete --all
+
+    django-admin galaxy-settings list
+    django-admin galaxy-settings list --raw
+
+    django-admin galaxy-settings inspect --key=foo
+
+    django-admin galaxy-settings update_cache
+    django-admin galaxy-settings clean_cache
+
+    django-admin galaxy-settings allowed_keys
+    """
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest='subcommand', required=True)
+
+        # Subcommand: set
+        set_parser = subparsers.add_parser('set', help='Set a Galaxy setting')
+        set_parser.add_argument('--key', required=True, help='Setting key')
+        set_parser.add_argument('--value', required=True, help='Setting value')
+        set_parser.add_argument('--is-secret', action='store_true', help='Mark as secret')
+
+        # Subcommand: get
+        get_parser = subparsers.add_parser('get', help='Get a Galaxy setting')
+        get_parser.add_argument('--key', required=True, help='Setting key')
+        get_parser.add_argument('--raw', action='store_true', help='Raw value from DB')
+        get_parser.add_argument('--default', help='Default value')
+
+        # Subcommand: delete
+        delete_parser = subparsers.add_parser('delete', help='Delete a Galaxy setting')
+        delete_parser.add_argument('--key', help='Setting key')
+        delete_parser.add_argument(
+            '--all-versions', action='store_true', help='Delete all versions'
+        )
+        delete_parser.add_argument('--all', action='store_true', help='Delete all settings')
+
+        # Subcommand: list
+        list_parser = subparsers.add_parser('list', help='List Galaxy settings')
+        list_parser.add_argument('--raw', action='store_true', help='Raw value from DB')
+
+        # Subcommand: inspect
+        inspect_parser = subparsers.add_parser('inspect', help='Inspect a Galaxy setting')
+        inspect_parser.add_argument('--key', required=True, help='Setting key')
+
+        # Subcommand: update_cache
+        subparsers.add_parser('update_cache', help='Update settings cache')
+
+        # Subcommand: clean_cache
+        subparsers.add_parser('clean_cache', help='Clean settings cache')
+
+        # Subcommand: allowed_keys
+        subparsers.add_parser('allowed_keys', help='List allowed settings keys')
+
+    def echo(self, message):
+        self.stdout.write(self.style.SUCCESS(str(message)))
+
+    def handle(self, *args, **options):
+        subcommand = options['subcommand']
+        result = getattr(self, f'handle_{subcommand}')(*args, **options)
+        self.echo(result)
+
+    def handle_set(self, *args, **options):
+        key = options['key']
+        value = options['value']
+        is_secret = options['is_secret']
+        return Setting.set_value_in_db(key, value, is_secret=is_secret)
+
+    def handle_get(self, *args, **options):
+        key = options['key']
+        raw = options['raw']
+        default = options['default']
+        if raw:
+            try:
+                return Setting.get_value_from_db(upperfy(key))
+            except Setting.DoesNotExist:
+                return default
+        return Setting.get(upperfy(key), default=default)
+
+    def handle_delete(self, *args, **options):
+        key = options['key']
+        all_versions = options['all_versions']
+        all_settings = options['all']
+        if all_settings:
+            result = Setting.objects.all().delete()
+            Setting.update_cache()
+            return result
+
+        with suppress(Setting.DoesNotExist):
+            if key and all_versions:
+                return Setting.delete_all_versions(upperfy(key))
+            if key:
+                return Setting.delete_latest_version(upperfy(key))
+
+        return "Nothing to delete"
+
+    def handle_list(self, *args, **options):
+        raw = options['raw']
+        data = Setting.as_dict()
+        if raw:
+            return data
+        return {k: Setting.get(k) for k in data}
+
+    def handle_inspect(self, *args, **options):
+        key = options['key']
+        from dynaconf.utils.inspect import get_history
+        return get_history(settings, key)
+
+    def handle_update_cache(self, *args, **options):
+        return Setting.update_cache()
+
+    def handle_clean_cache(self, *args, **options):
+        return Setting.clean_cache()
+
+    def handle_allowed_keys(self, *args, **options):
+        return DYNAMIC_SETTINGS_SCHEMA.keys()