Merge pull request #854 from camptocamp/fix-empty-env

Always use env_or_config or env_or_settings to correctly handle empt…

.prospector.yaml

@@ -21,6 +21,7 @@ pylint:
     - ungrouped-imports
     - unused-argument
     - use-symbolic-message-instead
+    - wrong-import-order
 
 pep8:
   options:

c2cwsgiutils/auth.py

@@ -5,7 +5,7 @@
 from pyramid.httpexceptions import HTTPForbidden
 
 # noinspection PyProtectedMember
-from c2cwsgiutils._utils import env_or_settings, env_or_config, config_bool
+from c2cwsgiutils.config_utils import config_bool, env_or_config, env_or_settings
 
 COOKIE_AGE = 7 * 24 * 3600
 SECRET_PROP = "c2c.secret"  # nosec  # noqa

c2cwsgiutils/broadcast/__init__.py

@@ -6,7 +6,8 @@
 from typing import Any, Callable, Dict, List, Optional
 
 import pyramid.config
-from c2cwsgiutils import _utils, redis_utils
+
+from c2cwsgiutils import config_utils, redis_utils
 from c2cwsgiutils.broadcast import interface, local, redis
 
 LOG = logging.getLogger(__name__)
@@ -21,8 +22,10 @@ def init(config: Optional[pyramid.config.Configurator] = None) -> None:
     Initialize the broadcaster with Redis, if configured. Otherwise, fall back to a fake local implementation.
     """
     global _broadcaster
-    broadcast_prefix = _utils.env_or_config(config, BROADCAST_ENV_KEY, BROADCAST_CONFIG_KEY, "broadcast_api_")
-    master, slave, _ = redis_utils.get()
+    broadcast_prefix = config_utils.env_or_config(
+        config, BROADCAST_ENV_KEY, BROADCAST_CONFIG_KEY, "broadcast_api_"
+    )
+    master, slave, _ = redis_utils.get(config.get_settings() if config else None)
     if _broadcaster is None:
         if master is not None and slave is not None:
             _broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave)

c2cwsgiutils/db_maintenance_view.py

@@ -2,7 +2,8 @@
 from typing import Any, Mapping, Optional, cast
 
 import pyramid.request
-from c2cwsgiutils import _utils, auth, broadcast, db, redis_utils
+
+from c2cwsgiutils import auth, broadcast, config_utils, db, redis_utils
 
 LOG = logging.getLogger(__name__)
 CONFIG_KEY = "c2c.db_maintenance_view_enabled"
@@ -16,7 +17,9 @@ def install_subscriber(config: pyramid.config.Configurator) -> None:
     """
     if auth.is_enabled(config, ENV_KEY, CONFIG_KEY):
         config.add_route(
-            "c2c_db_maintenance", _utils.get_base_path(config) + r"/db/maintenance", request_method="GET"
+            "c2c_db_maintenance",
+            config_utils.get_base_path(config) + r"/db/maintenance",
+            request_method="GET",
         )
         config.add_view(_db_maintenance, route_name="c2c_db_maintenance", renderer="fast_json", http_cache=0)
         _restore(config)
@@ -60,13 +63,13 @@ def _restore(config: pyramid.config.Configurator) -> None:
 
 
 def _store(settings: Mapping[str, Any], readonly: bool) -> None:
-    master, _, _ = redis_utils.get()
+    master, _, _ = redis_utils.get(settings)
     if master is not None:
         master.set(REDIS_PREFIX + "force_readonly", "true" if readonly else "false")
 
 
 def _get_redis_value(settings: Mapping[str, Any]) -> Optional[str]:
-    _, slave, _ = redis_utils.get()
+    _, slave, _ = redis_utils.get(settings)
     if slave is not None:
         value = slave.get(REDIS_PREFIX + "force_readonly")
         return str(value) if value else None

c2cwsgiutils/debug/__init__.py

@@ -2,7 +2,7 @@
 
 import pyramid.config
 
-from c2cwsgiutils import _utils, auth
+from c2cwsgiutils import auth, config_utils
 from c2cwsgiutils.debug import utils
 
 CONFIG_KEY = "c2c.debug_view_enabled"
@@ -26,7 +26,7 @@ def init_daemon(config: Optional[pyramid.config.Configurator] = None) -> None:
     Initialize the debug broadcast listeners. Used mostly for headless processes that depend on a master
     providing a normal REST API and broadcasting those requests.
     """
-    if _utils.env_or_config(config, ENV_KEY, CONFIG_KEY, type_=_utils.config_bool):
+    if config_utils.env_or_config(config, ENV_KEY, CONFIG_KEY, type_=config_utils.config_bool):
         from c2cwsgiutils.debug import _listeners
 
         _listeners.init()

c2cwsgiutils/debug/_views.py

@@ -1,18 +1,18 @@
-from datetime import datetime
 import gc
-from io import StringIO
 import logging
 import re
 import time
+from datetime import datetime
+from io import StringIO
 from typing import Any, Callable, Dict, List, Mapping
 
 import objgraph
 import pyramid.config
-from pyramid.httpexceptions import HTTPException, exception_response
 import pyramid.request
 import pyramid.response
+from pyramid.httpexceptions import HTTPException, exception_response
 
-from c2cwsgiutils import _utils, auth, broadcast
+from c2cwsgiutils import auth, broadcast, config_utils
 from c2cwsgiutils.debug.utils import dump_memory_maps, get_size
 
 LOG = logging.getLogger(__name__)
@@ -155,7 +155,7 @@ def _add_view(
     config: pyramid.config.Configurator, name: str, path: str, view: Callable[[pyramid.request.Request], Any]
 ) -> None:
     config.add_route(
-        "c2c_debug_" + name, _utils.get_base_path(config) + r"/debug/" + path, request_method="GET"
+        "c2c_debug_" + name, config_utils.get_base_path(config) + r"/debug/" + path, request_method="GET"
     )
     config.add_view(view, route_name="c2c_debug_" + name, renderer="fast_json", http_cache=0)
 

c2cwsgiutils/errors.py

@@ -12,7 +12,7 @@
 from pyramid.httpexceptions import HTTPError, HTTPException, HTTPRedirection, HTTPSuccessful
 from webob.request import DisconnectionError
 
-from c2cwsgiutils import _utils, auth
+from c2cwsgiutils import auth, config_utils
 
 DEVELOPMENT = os.environ.get("DEVELOPMENT", "0") != "0"
 DEPRECATED_CONFIG_KEY = "c2c.error_details_secret"
@@ -153,7 +153,9 @@ def _passthrough(exception: HTTPException, request: pyramid.request.Request) ->
 
 def init(config: pyramid.config.Configurator) -> None:
     if (
-        _utils.env_or_config(config, "C2C_ENABLE_EXCEPTION_HANDLING", "c2c.enable_exception_handling", "0")
+        config_utils.env_or_config(
+            config, "C2C_ENABLE_EXCEPTION_HANDLING", "c2c.enable_exception_handling", "0"
+        )
         != "0"
     ):
         for exception in (HTTPSuccessful, HTTPRedirection):

c2cwsgiutils/health_check.py

@@ -13,9 +13,8 @@
 import time
 import traceback
 from collections import Counter
-from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
-
 from enum import Enum
+from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 
 import pyramid.config
 import pyramid.request
@@ -24,7 +23,7 @@
 import sqlalchemy.orm
 from pyramid.httpexceptions import HTTPNotFound
 
-from c2cwsgiutils import _utils, auth, broadcast, redis_utils, stats, version
+from c2cwsgiutils import auth, broadcast, config_utils, redis_utils, stats, version
 
 LOG = logging.getLogger(__name__)
 ALEMBIC_HEAD_RE = re.compile(r"^([a-f0-9]+) \(head\)\n$")
@@ -93,13 +92,18 @@ class HealthCheck:
 
     def __init__(self, config: pyramid.config.Configurator) -> None:
         config.add_route(
-            "c2c_health_check", _utils.get_base_path(config) + r"/health_check", request_method="GET"
+            "c2c_health_check", config_utils.get_base_path(config) + r"/health_check", request_method="GET"
         )
         config.add_view(self._view, route_name="c2c_health_check", renderer="fast_json", http_cache=0)
         self._checks: List[Tuple[str, Callable[[pyramid.request.Request], Any], int]] = []
 
-        redis = os.environ.get(redis_utils.REDIS_SENTINELS_KEY, os.environ.get(redis_utils.REDIS_URL_KEY))
-        if redis:
+        self.name = config_utils.env_or_config(
+            config,
+            redis_utils.REDIS_SENTINELS_KEY,
+            redis_utils.REDIS_SENTINELS_KEY_PROP,
+            config_utils.env_or_config(config, redis_utils.REDIS_URL_KEY, redis_utils.REDIS_URL_KEY_PROP),
+        )
+        if self.name:
             self.add_redis_check(level=2)
             if version.get_version() is not None:
                 self.add_version_check(level=2)
@@ -248,7 +252,7 @@ def add_redis_check(self, name: Optional[str] = None, level: int = 1) -> None:
         """
 
         def check(request: pyramid.request.Request) -> Any:
-            master, slave, sentinel = redis_utils.get()
+            master, slave, sentinel = redis_utils.get(request.registry.settings)
 
             result = {}
 
@@ -264,7 +268,12 @@ def add(name: str, func: Callable[..., Any], *args: Any) -> None:
             if slave is not None:
                 add("slave_info", slave.info)
             if sentinel is not None:
-                service_name = os.environ.get(redis_utils.REDIS_SERVICENAME_KEY, "mymaster")
+                service_name = config_utils.env_or_settings(
+                    request.registry.settings,
+                    redis_utils.REDIS_SERVICENAME_KEY,
+                    redis_utils.REDIS_SERVICENAME_KEY_PROP,
+                    "mymaster",
+                )
                 add("sentinel", sentinel.sentinels[0].sentinel)
                 add("sentinel_masters", sentinel.sentinels[0].sentinel_masters)
                 add("sentinel_master", sentinel.sentinels[0].sentinel_master, service_name)
@@ -274,8 +283,11 @@ def add(name: str, func: Callable[..., Any], *args: Any) -> None:
             return result
 
         if name is None:
-            name = os.environ.get(redis_utils.REDIS_SENTINELS_KEY, os.environ.get(redis_utils.REDIS_URL_KEY))
-        assert name
+            name = self.name
+
+            if name is None:
+                raise RuntimeError("Redis should be confgured")
+
         self._checks.append((name, check, level))
 
     def add_version_check(self, name: str = "version", level: int = 2) -> None:

c2cwsgiutils/index.py

@@ -4,7 +4,7 @@
 import pyramid.request
 import pyramid.response
 
-from c2cwsgiutils import _utils, profiler
+from c2cwsgiutils import profiler, config_utils
 from c2cwsgiutils.auth import is_auth
 
 additional_title: Optional[str] = None
@@ -308,7 +308,7 @@ def _health_check(request: pyramid.request.Request) -> str:
 
 
 def init(config: pyramid.config.Configurator) -> None:
-    base_path = _utils.get_base_path(config)
+    base_path = config_utils.get_base_path(config)
     if base_path != "":
         config.add_route("c2c_index", base_path, request_method=("GET", "POST"))
         config.add_view(_index, route_name="c2c_index", http_cache=0)

c2cwsgiutils/logging_view.py

@@ -2,7 +2,8 @@
 from typing import Any, Generator, Mapping, Tuple
 
 import pyramid.request
-from c2cwsgiutils import _utils, auth, broadcast, redis_utils
+
+from c2cwsgiutils import auth, broadcast, redis_utils, config_utils
 
 LOG = logging.getLogger(__name__)
 CONFIG_KEY = "c2c.log_view_enabled"
@@ -16,7 +17,7 @@ def install_subscriber(config: pyramid.config.Configurator) -> None:
     """
     if auth.is_enabled(config, ENV_KEY, CONFIG_KEY):
         config.add_route(
-            "c2c_logging_level", _utils.get_base_path(config) + r"/logging/level", request_method="GET"
+            "c2c_logging_level", config_utils.get_base_path(config) + r"/logging/level", request_method="GET"
         )
         config.add_view(
             _logging_change_level, route_name="c2c_logging_level", renderer="fast_json", http_cache=0
@@ -67,15 +68,15 @@ def _restore_overrides(config: pyramid.config.Configurator) -> None:
 
 def _store_override(settings: Mapping[str, Any], name: str, level: str) -> None:
     try:
-        master, _, _ = redis_utils.get()
+        master, _, _ = redis_utils.get(settings)
         if master:
             master.set(REDIS_PREFIX + name, level)
     except ImportError:
         pass
 
 
 def _list_overrides(settings: Mapping[str, Any]) -> Generator[Tuple[str, str], None, None]:
-    _, slave, _ = redis_utils.get()
+    _, slave, _ = redis_utils.get(settings)
     if slave is not None:
         for key in slave.scan_iter(REDIS_PREFIX + "*"):
             level = slave.get(key)

c2cwsgiutils/pyramid_logging.py

@@ -13,7 +13,7 @@
 import os
 import socket
 import sys
-from typing import Any, MutableMapping, Mapping, IO, Optional
+from typing import IO, Any, Mapping, MutableMapping, Optional
 
 import cee_syslog_handler
 from pyramid.threadlocal import get_current_request

c2cwsgiutils/redis_stats.py

@@ -1,9 +1,9 @@
 import logging
-from typing import Optional, Callable, Any, Dict  # noqa  # pylint: disable=unused-import
+from typing import Any, Callable, Dict, Optional  # noqa  # pylint: disable=unused-import
 
 import pyramid.config
 
-from c2cwsgiutils import stats, _utils
+from c2cwsgiutils import config_utils, stats
 
 LOG = logging.getLogger(__name__)
 ORIG: Optional[Callable[..., Any]] = None
@@ -23,7 +23,9 @@ def _execute_command_patch(self: Any, *args: Any, **options: Any) -> Any:
 
 def init(config: Optional[pyramid.config.Configurator] = None) -> None:
     global ORIG
-    if _utils.env_or_config(config, "C2C_TRACK_REDIS", "c2c.track_redis", True, _utils.config_bool):
+    if config_utils.env_or_config(
+        config, "C2C_TRACK_REDIS", "c2c.track_redis", True, config_utils.config_bool
+    ):
         try:
             import redis.client
 

c2cwsgiutils/redis_utils.py

@@ -1,14 +1,15 @@
 import logging
-import os
 import threading
 import time
-from typing import Optional, Tuple
+from typing import Any, Mapping, Optional, Tuple
 
 import redis.client
 import redis.exceptions
 import redis.sentinel
 import yaml
 
+import c2cwsgiutils.config_utils
+
 LOG = logging.getLogger(__name__)
 
 REDIS_URL_KEY = "C2C_REDIS_URL"
@@ -17,24 +18,39 @@
 REDIS_SERVICENAME_KEY = "C2C_REDIS_SERVICENAME"
 REDIS_DB_KEY = "C2C_REDIS_DB"
 
+REDIS_URL_KEY_PROP = "c2c.redis_url"
+REDIS_OPTIONS_KEY_PROP = "c2c.redis_options"
+REDIS_SENTINELS_KEY_PROP = "c2c.redis_sentinels"
+REDIS_SERVICENAME_KEY_PROP = "c2c.redis_servicename"
+REDIS_DB_KEY_PROP = "c2c.redis_db"
+
 _master: Optional[redis.Redis] = None
 _slave: Optional[redis.Redis] = None
 _sentinel: Optional[redis.sentinel.Sentinel] = None
 
 
-def get() -> Tuple[Optional[redis.Redis], Optional[redis.Redis], Optional[redis.sentinel.Sentinel]]:
+def get(
+    settings: Optional[Mapping[str, Any]] = None,
+) -> Tuple[Optional[redis.Redis], Optional[redis.Redis], Optional[redis.sentinel.Sentinel]]:
     if _master is None:
-        _init()
+        _init(settings)
     return _master, _slave, _sentinel
 
 
-def _init() -> None:
+def _init(settings: Optional[Mapping[str, Any]]) -> None:
     global _master, _slave, _sentinel
-    sentinels = os.environ.get(REDIS_SENTINELS_KEY)
-    service_name = os.environ.get(REDIS_SERVICENAME_KEY)
-    db = os.environ.get(REDIS_DB_KEY)
-    url = os.environ.get(REDIS_URL_KEY)
-    redis_options_ = os.environ.get(REDIS_OPTIONS_KEY)
+    sentinels = c2cwsgiutils.config_utils.env_or_settings(
+        settings, REDIS_SENTINELS_KEY, REDIS_SENTINELS_KEY_PROP
+    )
+    service_name = c2cwsgiutils.config_utils.env_or_settings(
+        settings, REDIS_SERVICENAME_KEY, REDIS_SERVICENAME_KEY_PROP
+    )
+    db = c2cwsgiutils.config_utils.env_or_settings(settings, REDIS_DB_KEY, REDIS_DB_KEY_PROP)
+    url = c2cwsgiutils.config_utils.env_or_settings(settings, REDIS_URL_KEY, REDIS_URL_KEY_PROP)
+    redis_options_ = c2cwsgiutils.config_utils.env_or_settings(
+        settings, REDIS_OPTIONS_KEY, REDIS_OPTIONS_KEY_PROP
+    )
+
     redis_options = (
         {}
         if redis_options_ is None