From cea27fab7322226568901ce5a8fdf6de83d174b4 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Thu, 6 Jun 2019 11:29:18 +0200 Subject: [PATCH 01/27] Support source on Variable --- ecological/config.py | 29 +++++++++++++++++++++++++---- tests/test_config.py | 10 ++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 3290055..2175841 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -4,8 +4,23 @@ import enum import os import warnings -from typing import (Any, AnyStr, ByteString, Callable, Dict, FrozenSet, List, - Optional, Set, Tuple, Type, TypeVar, Union, get_type_hints) +from typing import ( + Any, + AnyStr, + ByteString, + Callable, + Dict, + FrozenSet, + List, + NewType, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + get_type_hints, +) try: from typing import GenericMeta @@ -105,6 +120,10 @@ def cast(representation: str, wanted_type: type): return wanted_type(representation) +VariableName = NewType("VariableName", Union[str, bytes]) +VariableValue = NewType("VariableValue", Union[str, bytes]) + + class Variable: """ Class to handle specific properties @@ -112,10 +131,11 @@ class Variable: def __init__( self, - variable_name: str, + variable_name: VariableName, default=_NO_DEFAULT, *, transform: Callable[[str, type], Any] = cast, + source: Dict[VariableName, VariableValue] = os.environ, ): """ :param variable_name: Environment variable to get @@ -125,6 +145,7 @@ def __init__( self.name = variable_name self.default = default self.transform = transform + self.source = source def get(self, wanted_type: WantedType) -> Union[WantedType, Any]: """ @@ -140,7 +161,7 @@ def get(self, wanted_type: WantedType) -> Union[WantedType, Any]: :return: value as wanted_type """ try: - raw_value = os.environ[self.name] + raw_value = self.source[self.name] except KeyError: if self.default is _NO_DEFAULT: raise AttributeError(f"Configuration error: '{self.name}' is not set.") diff --git a/tests/test_config.py b/tests/test_config.py index 7baaf6d..6b31270 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +import os import typing import pytest @@ -203,3 +204,12 @@ def test_config_autoload_is_ignored_on_autoconfig(): class Configuration(ecological.AutoConfig, autoload=ecological.Autoload.NEVER): my_var1: str + + +def test_variable_is_loaded_from_source(monkeypatch, base_class): + monkeypatch.setitem(os.environb, b"A_BYTES", b"a-bytes-value") + + class Configuration(base_class): + a_bytes: bytes = ecological.Variable(b"A_BYTES", source=os.environb) + + assert Configuration.a_bytes == b"a-bytes-value" From 57445425314bfcc1c091dbd48e2c14a1e10be467 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Thu, 6 Jun 2019 17:33:15 +0200 Subject: [PATCH 02/27] WIP --- ecological/config.py | 71 ++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 2175841..8998b5b 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -38,7 +38,6 @@ except ImportError: Deque = Counter = _NOT_IMPORTED -_NO_DEFAULT = object() TYPES_THAT_NEED_TO_BE_PARSED = [bool, list, set, tuple, dict] TYPING_TO_REGULAR_TYPE = { AnyStr: str, @@ -122,46 +121,50 @@ def cast(representation: str, wanted_type: type): VariableName = NewType("VariableName", Union[str, bytes]) VariableValue = NewType("VariableValue", Union[str, bytes]) +Source = NewType("Source", Dict[VariableName, VariableValue]) +TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) +Prefix = NewType("Prefix", VariableName) +_NO_DEFAULT = object() +@dataclasses.dataclass class Variable: """ - Class to handle specific properties + """ - def __init__( + name: VariableName + default: Any = _NO_DEFAULT + transform: Optional[TransformCallable] = None + source: Optional[Source] = None + prefix: Optional[Prefix] = None + + def finalize( self, - variable_name: VariableName, - default=_NO_DEFAULT, *, - transform: Callable[[str, type], Any] = cast, - source: Dict[VariableName, VariableValue] = os.environ, + prefix: Optional[Prefix] = None, + transform: TransformCallable, + source: Source, ): - """ - :param variable_name: Environment variable to get - :param default: Default value. - :param transform: function to convert the env string to the wanted type - """ - self.name = variable_name - self.default = default - self.transform = transform - self.source = source + if not self.prefix and prefix: + separator = {str: "_", bytes: b"_"} + self.name = prefix + separator[type(prefix)] + self.name + self.name = self.name.upper() + self.transform = self.transform or transform + self.source = self.source or source def get(self, wanted_type: WantedType) -> Union[WantedType, Any]: """ - Gets ``self.variable_name`` from the environment and tries to cast it to ``wanted_type``. + Gets ``self.name`` from the environment and tries to cast it to ``wanted_type``. If ``self.default`` is ``_NO_DEFAULT`` and the env variable is not set this will raise an ``AttributeError``, if the ``self.default`` is set to something else, its value will be returned. If casting fails, this function will raise a ``ValueError``. - - :param wanted_type: type to return - :return: value as wanted_type """ try: - raw_value = self.source[self.name] + raw_value = self.source[self.name.upper()] except KeyError: if self.default is _NO_DEFAULT: raise AttributeError(f"Configuration error: '{self.name}' is not set.") @@ -200,6 +203,8 @@ class _Options: prefix: Optional[str] = None autoload: Autoload = Autoload.CLASS + source: Source = os.environ + transform: TransformCallable = cast @classmethod def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": @@ -290,7 +295,6 @@ def load(cls: "Config", target_obj: Optional[object] = None): target_obj = target_obj or cls annotations: Dict[str, type] = get_type_hints(cls) attribute_dict = vars(cls).copy() - prefix = cls._options.prefix # Add attributes without defaults to the the attribute dict attribute_dict.update( @@ -302,21 +306,19 @@ def load(cls: "Config", target_obj: Optional[object] = None): ) for attribute_name, default_value in attribute_dict.items(): - if attribute_name.startswith("_"): - # private attributes are not changed + if attribute_name.startswith("_") or isinstance(default_value, cls): + # private attributes and nested configs are not changed continue + if isinstance(default_value, Variable): attribute = default_value - elif isinstance(default_value, Config): - # passthrough for nested configs - setattr(target_obj, attribute_name, default_value) - continue else: - if prefix: - env_variable_name = f"{prefix}_{attribute_name}".upper() - else: - env_variable_name = attribute_name.upper() - attribute = Variable(env_variable_name, default_value) + attribute = Variable(attribute_name, default_value) + attribute.finalize( + prefix=cls._options.prefix, + transform=cls._options.transform, + source=cls._options.source, + ) attribute_type = annotations.get( attribute_name, str @@ -333,3 +335,6 @@ def __init_subclass__(cls, prefix: Optional[str] = None, **kwargs): DeprecationWarning, ) super().__init_subclass__(prefix=prefix, autoload=Autoload.CLASS) + + +dataclasses.field From 53f644779358a0eb2875d158beb8c18d5dacb330 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Fri, 7 Jun 2019 16:28:45 +0200 Subject: [PATCH 03/27] Factor get from variable out, simplify load method --- ecological/config.py | 154 ++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 89 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 8998b5b..7f2dc9f 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -119,66 +119,6 @@ def cast(representation: str, wanted_type: type): return wanted_type(representation) -VariableName = NewType("VariableName", Union[str, bytes]) -VariableValue = NewType("VariableValue", Union[str, bytes]) -Source = NewType("Source", Dict[VariableName, VariableValue]) -TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) -Prefix = NewType("Prefix", VariableName) -_NO_DEFAULT = object() - - -@dataclasses.dataclass -class Variable: - """ - - """ - - name: VariableName - default: Any = _NO_DEFAULT - transform: Optional[TransformCallable] = None - source: Optional[Source] = None - prefix: Optional[Prefix] = None - - def finalize( - self, - *, - prefix: Optional[Prefix] = None, - transform: TransformCallable, - source: Source, - ): - if not self.prefix and prefix: - separator = {str: "_", bytes: b"_"} - self.name = prefix + separator[type(prefix)] + self.name - self.name = self.name.upper() - self.transform = self.transform or transform - self.source = self.source or source - - def get(self, wanted_type: WantedType) -> Union[WantedType, Any]: - """ - Gets ``self.name`` from the environment and tries to cast it to ``wanted_type``. - - If ``self.default`` is ``_NO_DEFAULT`` and the env variable is not set this will raise an - ``AttributeError``, if the ``self.default`` is set to something else, its value will be - returned. - - If casting fails, this function will raise a ``ValueError``. - """ - try: - raw_value = self.source[self.name.upper()] - except KeyError: - if self.default is _NO_DEFAULT: - raise AttributeError(f"Configuration error: '{self.name}' is not set.") - else: - return self.default - - try: - value = self.transform(raw_value, wanted_type) - except (ValueError, SyntaxError) as e: - raise ValueError(f"Invalid configuration for '{self.name}': {e}.") - - return value - - class Autoload(enum.Enum): """ Represents different approaches to the attribute values @@ -194,6 +134,13 @@ class Autoload(enum.Enum): NEVER = "NEVER" +VariableName = NewType("VariableName", Union[str, bytes]) +VariableValue = NewType("VariableValue", Union[str, bytes]) +Source = NewType("Source", Dict[VariableName, VariableValue]) +TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) +_NO_DEFAULT = object() + + @dataclasses.dataclass class _Options: """ @@ -228,6 +175,21 @@ def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": ) from e +class Variable: + def __init__( + self, + variable_name: Optional[VariableName] = None, + default: Any = _NO_DEFAULT, + *, + transform: Optional[TransformCallable] = None, + source: Optional[Source] = None, + ): + self.name = variable_name + self.default = default + self.transform = transform + self.source = source + + class Config: """ When ``Config`` subclasses are created, by default ``Ecological`` will set their @@ -271,6 +233,39 @@ def __init__(self, *args, **kwargs): if cls._options.autoload is Autoload.OBJECT: cls.load(self) + @classmethod + def fetch_source_value( + cls: "Config", attr_name: str, attr_value: Any, attr_type: type + ): + if isinstance(attr_value, Variable): + source_name = attr_value.name + default = attr_value.default + transform = attr_value.transform or cls._options.transform + source = attr_value.source or cls._options.source + else: + source_name = attr_name + if cls._options.prefix: + source_name = f"{cls._options.prefix}_{source_name}" + source_name = source_name.upper() + default = attr_value + transform = cls._options.transform + source = cls._options.source + + try: + raw_value = source[source_name] + except KeyError as e: + if default is _NO_DEFAULT: + raise AttributeError( + f"Configuration error: '{source_name}' is not set." + ) from e + else: + return default + + try: + return transform(raw_value, attr_type) + except (ValueError, SyntaxError) as e: + raise ValueError(f"Invalid configuration for '{source_name}': {e}.") + @classmethod def load(cls: "Config", target_obj: Optional[object] = None): """ @@ -293,38 +288,19 @@ def load(cls: "Config", target_obj: Optional[object] = None): (the default value or ``_NO_DEFAULT``) and do the same process as in the previous point. """ target_obj = target_obj or cls - annotations: Dict[str, type] = get_type_hints(cls) - attribute_dict = vars(cls).copy() - - # Add attributes without defaults to the the attribute dict - attribute_dict.update( - { - attribute_name: _NO_DEFAULT - for attribute_name in annotations.keys() - if attribute_name not in attribute_dict - } - ) + cls_dict = vars(cls).copy() + attr_types: Dict[str, type] = get_type_hints(cls) + attr_names = set(cls_dict).union(attr_types.keys()) - for attribute_name, default_value in attribute_dict.items(): - if attribute_name.startswith("_") or isinstance(default_value, cls): - # private attributes and nested configs are not changed + for attr_name in attr_names: + if attr_name.startswith("_") or isinstance(attr_name, cls): continue - if isinstance(default_value, Variable): - attribute = default_value - else: - attribute = Variable(attribute_name, default_value) - attribute.finalize( - prefix=cls._options.prefix, - transform=cls._options.transform, - source=cls._options.source, - ) - - attribute_type = annotations.get( - attribute_name, str - ) # by default attributes are strings - value = attribute.get(attribute_type) - setattr(target_obj, attribute_name, value) + attr_value = cls_dict.get(attr_name, _NO_DEFAULT) + attr_type = attr_types.get(attr_name, str) + + source_value = cls.fetch_source_value(attr_name, attr_value, attr_type) + setattr(target_obj, attr_name, source_value) # DEPRECATED: For backward compatibility purposes only From 8fcfa30a57a98a0adf563b349ce3b0ace85c2193 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Fri, 7 Jun 2019 16:30:26 +0200 Subject: [PATCH 04/27] Formatting --- ecological/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecological/config.py b/ecological/config.py index 7f2dc9f..87521ec 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -247,6 +247,7 @@ def fetch_source_value( if cls._options.prefix: source_name = f"{cls._options.prefix}_{source_name}" source_name = source_name.upper() + default = attr_value transform = cls._options.transform source = cls._options.source From fbc22481259157de9483098bc6ae557fa5b1cff3 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Fri, 7 Jun 2019 16:37:40 +0200 Subject: [PATCH 05/27] Factor casting to separate module --- ecological/casting.py | 97 ++++++++++++++++++++++++++ ecological/config.py | 159 ++++++------------------------------------ 2 files changed, 119 insertions(+), 137 deletions(-) create mode 100644 ecological/casting.py diff --git a/ecological/casting.py b/ecological/casting.py new file mode 100644 index 0000000..5156edc --- /dev/null +++ b/ecological/casting.py @@ -0,0 +1,97 @@ +import ast +import collections +from typing import AnyStr, ByteString, Dict, FrozenSet, List, Set, Tuple + +try: + from typing import GenericMeta + + PEP560 = False +except ImportError: + GenericMeta = None + PEP560 = True + + +_NOT_IMPORTED = object() + +try: # Types added in Python 3.6.1 + from typing import Counter, Deque +except ImportError: + Deque = Counter = _NOT_IMPORTED + +TYPES_THAT_NEED_TO_BE_PARSED = [bool, list, set, tuple, dict] +TYPING_TO_REGULAR_TYPE = { + AnyStr: str, + ByteString: bytes, + Counter: collections.Counter, + Deque: collections.deque, + Dict: dict, + FrozenSet: frozenset, + List: list, + Set: set, + Tuple: tuple, +} + + +def _cast_typing_old(wanted_type: type) -> type: + """ + Casts typing types in Python < 3.7 + """ + wanted_type = TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type) + if isinstance(wanted_type, GenericMeta): + # Fallback to try to map complex typing types to real types + for base in wanted_type.__bases__: + # if not isinstance(base, Generic): + # # If it's not a Generic class then it can be a real type + # wanted_type = base + # break + if base in TYPING_TO_REGULAR_TYPE: + # The mapped type in bases is most likely the base type for complex types + # (for example List[int]) + wanted_type = TYPING_TO_REGULAR_TYPE[base] + break + return wanted_type + + +def _cast_typing_pep560(wanted_type: type) -> type: + """ + Casts typing types in Python >= 3.7 + See https://www.python.org/dev/peps/pep-0560/ + """ + # If it's in the dict in Python >= 3.7 we know how to handle it + if wanted_type in TYPING_TO_REGULAR_TYPE: + return TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type) + + try: + return wanted_type.__origin__ + except AttributeError: # This means it's (probably) not a typing type + return wanted_type + + +def cast(representation: str, wanted_type: type): + """ + Casts the string ``representation`` to the ``wanted type``. + This function also supports some ``typing`` types by mapping them to 'real' types. + + Some types, like ``bool`` and ``list``, need to be parsed with ast. + """ + # The only distinguishing feature of NewType (both before and after PEP560) + # is its __supertype__ field, which it is the only "typing" member to have. + # Since newtypes can be nested, we process __supertype__ as long as available. + while hasattr(wanted_type, "__supertype__"): + wanted_type = wanted_type.__supertype__ + + # If it's another typing type replace it with the real type + if PEP560: # python >= 3.7 + wanted_type = _cast_typing_pep560(wanted_type) + else: + wanted_type = _cast_typing_old(wanted_type) + + if wanted_type in TYPES_THAT_NEED_TO_BE_PARSED: + value = ( + ast.literal_eval(representation) + if isinstance(representation, str) + else representation + ) + return wanted_type(value) + else: + return wanted_type(representation) diff --git a/ecological/config.py b/ecological/config.py index 87521ec..919dcf3 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -1,122 +1,32 @@ -import ast -import collections import dataclasses import enum import os import warnings -from typing import ( - Any, - AnyStr, - ByteString, - Callable, - Dict, - FrozenSet, - List, - NewType, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - get_type_hints, -) - -try: - from typing import GenericMeta - - PEP560 = False -except ImportError: - GenericMeta = None - PEP560 = True - - -_NOT_IMPORTED = object() - -try: # Types added in Python 3.6.1 - from typing import Counter, Deque -except ImportError: - Deque = Counter = _NOT_IMPORTED - -TYPES_THAT_NEED_TO_BE_PARSED = [bool, list, set, tuple, dict] -TYPING_TO_REGULAR_TYPE = { - AnyStr: str, - ByteString: bytes, - Counter: collections.Counter, - Deque: collections.deque, - Dict: dict, - FrozenSet: frozenset, - List: list, - Set: set, - Tuple: tuple, -} - -WantedType = TypeVar("WantedType") - - -def _cast_typing_old(wanted_type: type) -> type: - """ - Casts typing types in Python < 3.7 - """ - wanted_type = TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type) - if isinstance(wanted_type, GenericMeta): - # Fallback to try to map complex typing types to real types - for base in wanted_type.__bases__: - # if not isinstance(base, Generic): - # # If it's not a Generic class then it can be a real type - # wanted_type = base - # break - if base in TYPING_TO_REGULAR_TYPE: - # The mapped type in bases is most likely the base type for complex types - # (for example List[int]) - wanted_type = TYPING_TO_REGULAR_TYPE[base] - break - return wanted_type - - -def _cast_typing_pep560(wanted_type: type) -> type: - """ - Casts typing types in Python >= 3.7 - See https://www.python.org/dev/peps/pep-0560/ - """ - # If it's in the dict in Python >= 3.7 we know how to handle it - if wanted_type in TYPING_TO_REGULAR_TYPE: - return TYPING_TO_REGULAR_TYPE.get(wanted_type, wanted_type) +from typing import (Any, Callable, Dict, NewType, Optional, Union, + get_type_hints) - try: - return wanted_type.__origin__ - except AttributeError: # This means it's (probably) not a typing type - return wanted_type +from . import casting +VariableName = NewType("VariableName", Union[str, bytes]) +VariableValue = NewType("VariableValue", Union[str, bytes]) +Source = NewType("Source", Dict[VariableName, VariableValue]) +TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) +_NO_DEFAULT = object() -def cast(representation: str, wanted_type: type): - """ - Casts the string ``representation`` to the ``wanted type``. - This function also supports some ``typing`` types by mapping them to 'real' types. - Some types, like ``bool`` and ``list``, need to be parsed with ast. - """ - # The only distinguishing feature of NewType (both before and after PEP560) - # is its __supertype__ field, which it is the only "typing" member to have. - # Since newtypes can be nested, we process __supertype__ as long as available. - while hasattr(wanted_type, "__supertype__"): - wanted_type = wanted_type.__supertype__ - - # If it's another typing type replace it with the real type - if PEP560: # python >= 3.7 - wanted_type = _cast_typing_pep560(wanted_type) - else: - wanted_type = _cast_typing_old(wanted_type) - - if wanted_type in TYPES_THAT_NEED_TO_BE_PARSED: - value = ( - ast.literal_eval(representation) - if isinstance(representation, str) - else representation - ) - return wanted_type(value) - else: - return wanted_type(representation) +class Variable: + def __init__( + self, + variable_name: Optional[VariableName] = None, + default: Any = _NO_DEFAULT, + *, + transform: Optional[TransformCallable] = None, + source: Optional[Source] = None, + ): + self.name = variable_name + self.default = default + self.transform = transform + self.source = source class Autoload(enum.Enum): @@ -134,13 +44,6 @@ class Autoload(enum.Enum): NEVER = "NEVER" -VariableName = NewType("VariableName", Union[str, bytes]) -VariableValue = NewType("VariableValue", Union[str, bytes]) -Source = NewType("Source", Dict[VariableName, VariableValue]) -TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) -_NO_DEFAULT = object() - - @dataclasses.dataclass class _Options: """ @@ -151,7 +54,7 @@ class _Options: prefix: Optional[str] = None autoload: Autoload = Autoload.CLASS source: Source = os.environ - transform: TransformCallable = cast + transform: TransformCallable = casting.cast @classmethod def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": @@ -175,21 +78,6 @@ def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": ) from e -class Variable: - def __init__( - self, - variable_name: Optional[VariableName] = None, - default: Any = _NO_DEFAULT, - *, - transform: Optional[TransformCallable] = None, - source: Optional[Source] = None, - ): - self.name = variable_name - self.default = default - self.transform = transform - self.source = source - - class Config: """ When ``Config`` subclasses are created, by default ``Ecological`` will set their @@ -312,6 +200,3 @@ def __init_subclass__(cls, prefix: Optional[str] = None, **kwargs): DeprecationWarning, ) super().__init_subclass__(prefix=prefix, autoload=Autoload.CLASS) - - -dataclasses.field From be0b720652afaff9bfbbf2d2a851c2f32804d03b Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Fri, 7 Jun 2019 16:46:48 +0200 Subject: [PATCH 06/27] Trailing whitespace --- ecological/config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 919dcf3..5362367 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -2,8 +2,7 @@ import enum import os import warnings -from typing import (Any, Callable, Dict, NewType, Optional, Union, - get_type_hints) +from typing import Any, Callable, Dict, NewType, Optional, Union, get_type_hints from . import casting @@ -47,8 +46,8 @@ class Autoload(enum.Enum): @dataclasses.dataclass class _Options: """ - Acts as a container for metaclass keyword arguments provided during - ``Config`` class creation. + Acts as a container for metaclass keyword arguments provided during + ``Config`` class creation. """ prefix: Optional[str] = None From 624b530ec4b71e2fdb28322f767ff0a46ef00596 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 11 Jun 2019 16:07:39 +0200 Subject: [PATCH 07/27] Add tests for new global options --- ecological/config.py | 123 +++++++++++++++++++++---------------------- tests/test_config.py | 31 +++++++++++ 2 files changed, 91 insertions(+), 63 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 5362367..4a19e5f 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -2,30 +2,15 @@ import enum import os import warnings -from typing import Any, Callable, Dict, NewType, Optional, Union, get_type_hints +from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints from . import casting +_NO_DEFAULT = object() VariableName = NewType("VariableName", Union[str, bytes]) VariableValue = NewType("VariableValue", Union[str, bytes]) Source = NewType("Source", Dict[VariableName, VariableValue]) -TransformCallable = NewType("TransformCallable", Callable[[VariableValue, type], Any]) -_NO_DEFAULT = object() - - -class Variable: - def __init__( - self, - variable_name: Optional[VariableName] = None, - default: Any = _NO_DEFAULT, - *, - transform: Optional[TransformCallable] = None, - source: Optional[Source] = None, - ): - self.name = variable_name - self.default = default - self.transform = transform - self.source = source +TransformCallable = NewType("TransformCallable", Callable[[VariableValue, Type], Any]) class Autoload(enum.Enum): @@ -46,7 +31,7 @@ class Autoload(enum.Enum): @dataclasses.dataclass class _Options: """ - Acts as a container for metaclass keyword arguments provided during + Acts as the container for metaclass keyword arguments provided during ``Config`` class creation. """ @@ -54,16 +39,17 @@ class _Options: autoload: Autoload = Autoload.CLASS source: Source = os.environ transform: TransformCallable = casting.cast + wanted_type: Type = str @classmethod - def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": + def from_dict(cls, options_dict: Dict) -> "_Options": """ Produces ``_Options`` instance from given dictionary. - Items are deleted from ``metaclass_kwargs`` as a side-effect. + Items are deleted from ``options_dict`` as a side-effect. """ options_kwargs = {} for field in dataclasses.fields(cls): - value = metaclass_kwargs.pop(field.name, None) + value = options_dict.pop(field.name, None) if value is None: continue @@ -73,10 +59,42 @@ def from_metaclass_kwargs(cls, metaclass_kwargs: Dict) -> "Options": return cls(**options_kwargs) except TypeError as e: raise ValueError( - f"Invalid options for Config class: {metaclass_kwargs}." + f"Invalid options for Config class: {options_dict}." ) from e +@dataclasses.dataclass +class Variable: + variable_name: VariableName + default: Any = _NO_DEFAULT + transform: Optional[TransformCallable] = None + source: Optional[Source] = None + wanted_type: Type = dataclasses.field(init=False) + + def set_defaults( + self, *, transform: TransformCallable, source: Source, wanted_type: Type + ): + self.transform = self.transform or transform + self.source = self.source or source + self.wanted_type = wanted_type + + def get(self) -> VariableValue: + try: + raw_value = self.source[self.variable_name] + except KeyError as e: + if self.default is _NO_DEFAULT: + raise AttributeError( + f"Configuration error: '{self.variable_name}' is not set." + ) from e + else: + return self.default + + try: + return self.transform(raw_value, self.wanted_type) + except (ValueError, SyntaxError) as e: + raise ValueError(f"Invalid configuration for '{self.variable_name}': {e}.") + + class Config: """ When ``Config`` subclasses are created, by default ``Ecological`` will set their @@ -94,7 +112,7 @@ class Configuration(ecological.Config): It is possible to defer the calculation of attribute values by specifying the ``autoload`` keyword argument on your class definition. For possible strategies see the ``Autoload`` class definition. - + Caveats and Known Limitations ============================= @@ -109,7 +127,7 @@ class Configuration(ecological.Config): _options: _Options def __init_subclass__(cls, **kwargs): - cls._options = _Options.from_metaclass_kwargs(kwargs) + cls._options = _Options.from_dict(kwargs) super().__init_subclass__(**kwargs) if cls._options.autoload is Autoload.CLASS: cls.load(cls) @@ -120,40 +138,6 @@ def __init__(self, *args, **kwargs): if cls._options.autoload is Autoload.OBJECT: cls.load(self) - @classmethod - def fetch_source_value( - cls: "Config", attr_name: str, attr_value: Any, attr_type: type - ): - if isinstance(attr_value, Variable): - source_name = attr_value.name - default = attr_value.default - transform = attr_value.transform or cls._options.transform - source = attr_value.source or cls._options.source - else: - source_name = attr_name - if cls._options.prefix: - source_name = f"{cls._options.prefix}_{source_name}" - source_name = source_name.upper() - - default = attr_value - transform = cls._options.transform - source = cls._options.source - - try: - raw_value = source[source_name] - except KeyError as e: - if default is _NO_DEFAULT: - raise AttributeError( - f"Configuration error: '{source_name}' is not set." - ) from e - else: - return default - - try: - return transform(raw_value, attr_type) - except (ValueError, SyntaxError) as e: - raise ValueError(f"Invalid configuration for '{source_name}': {e}.") - @classmethod def load(cls: "Config", target_obj: Optional[object] = None): """ @@ -183,12 +167,25 @@ def load(cls: "Config", target_obj: Optional[object] = None): for attr_name in attr_names: if attr_name.startswith("_") or isinstance(attr_name, cls): continue - attr_value = cls_dict.get(attr_name, _NO_DEFAULT) - attr_type = attr_types.get(attr_name, str) + attr_type = attr_types.get(attr_name, cls._options.wanted_type) - source_value = cls.fetch_source_value(attr_name, attr_value, attr_type) - setattr(target_obj, attr_name, source_value) + if isinstance(attr_value, Variable): + variable = attr_value + else: + if cls._options.prefix: + prefix = cls._options.prefix + "_" + else: + prefix = "" + variable_name = (prefix + attr_name).upper() + variable = Variable(variable_name, attr_value) + variable.set_defaults( + transform=cls._options.transform, + source=cls._options.source, + wanted_type=attr_type, + ) + + setattr(target_obj, attr_name, variable.get()) # DEPRECATED: For backward compatibility purposes only diff --git a/tests/test_config.py b/tests/test_config.py index 6b31270..56c4bc7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -213,3 +213,34 @@ class Configuration(base_class): a_bytes: bytes = ecological.Variable(b"A_BYTES", source=os.environb) assert Configuration.a_bytes == b"a-bytes-value" + + +def test_global_transform_option_is_used_as_default(monkeypatch): + monkeypatch.setenv("IMPLICIT", "a") + monkeypatch.setenv("VAR_WITH_TRANSFORM", "b") + monkeypatch.setenv("VAR_WITHOUT_TRANSFORM", "c") + + class Configuration(ecological.Config, transform=lambda *args: "GLOBAL_TRANSFORM"): + implicit: str + var_without_transform = ecological.Variable("VAR_WITHOUT_TRANSFORM") + var_with_transform: int = ecological.Variable( + "VAR_WITH_TRANSFORM", transform=lambda *args: "VAR_TRANSFORM" + ) + + assert Configuration.implicit == "GLOBAL_TRANSFORM" + assert Configuration.var_with_transform == "VAR_TRANSFORM" + assert Configuration.var_without_transform == "GLOBAL_TRANSFORM" + + +def test_global_source_option_is_used_as_default(monkeypatch): + my_dict = {"IMPLICIT": "a", "VAR_WITHOUT_SOURCE": "b"} + monkeypatch.setenv("VAR_WITH_SOURCE", "c") + + class Configuration(ecological.Config, source=my_dict): + implicit: str + var_without_source = ecological.Variable("VAR_WITHOUT_SOURCE") + var_with_source = ecological.Variable("VAR_WITH_SOURCE", source=os.environ) + + assert Configuration.implicit == "a" + assert Configuration.var_without_source == "b" + assert Configuration.var_with_source == "c" From 00a4ef786be6d1e3233b1c9921c94c578ee21ff5 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 11 Jun 2019 16:42:59 +0200 Subject: [PATCH 08/27] Name is now flexible as well --- ecological/config.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 4a19e5f..b885f82 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -2,7 +2,8 @@ import enum import os import warnings -from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints +from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, + get_type_hints) from . import casting @@ -28,6 +29,15 @@ class Autoload(enum.Enum): NEVER = "NEVER" +def environ_name(attr_name: str, prefix: Optional[str] = None): + variable_name = "" + if prefix: + variable_name += f"{prefix}_" + variable_name += attr_name + + return variable_name.upper() + + @dataclasses.dataclass class _Options: """ @@ -40,6 +50,7 @@ class _Options: source: Source = os.environ transform: TransformCallable = casting.cast wanted_type: Type = str + variable_name: Callable[[str, Optional[str]], VariableName] = environ_name @classmethod def from_dict(cls, options_dict: Dict) -> "_Options": @@ -65,15 +76,21 @@ def from_dict(cls, options_dict: Dict) -> "_Options": @dataclasses.dataclass class Variable: - variable_name: VariableName + variable_name: Optional[VariableName] = None default: Any = _NO_DEFAULT transform: Optional[TransformCallable] = None source: Optional[Source] = None wanted_type: Type = dataclasses.field(init=False) def set_defaults( - self, *, transform: TransformCallable, source: Source, wanted_type: Type + self, + *, + variable_name: VariableName, + transform: TransformCallable, + source: Source, + wanted_type: Type, ): + self.variable_name = self.variable_name or variable_name self.transform = self.transform or transform self.source = self.source or source self.wanted_type = wanted_type @@ -173,13 +190,11 @@ def load(cls: "Config", target_obj: Optional[object] = None): if isinstance(attr_value, Variable): variable = attr_value else: - if cls._options.prefix: - prefix = cls._options.prefix + "_" - else: - prefix = "" - variable_name = (prefix + attr_name).upper() - variable = Variable(variable_name, attr_value) + variable = Variable(default=attr_value) variable.set_defaults( + variable_name=cls._options.variable_name( + attr_name, prefix=cls._options.prefix + ), transform=cls._options.transform, source=cls._options.source, wanted_type=attr_type, From 8f01f1e54e3f1a2ac5e2d5338687884e71ea4602 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 11 Jun 2019 16:46:18 +0200 Subject: [PATCH 09/27] Add friendly helper when using environb --- ecological/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ecological/config.py b/ecological/config.py index b885f82..73bbcbc 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -38,6 +38,10 @@ def environ_name(attr_name: str, prefix: Optional[str] = None): return variable_name.upper() +def environb_name(*args, **kwargs): + return environ_name(*args, **kwargs).encode() + + @dataclasses.dataclass class _Options: """ From 3e63b5aa54b82b734dea8d43de1dcb697d69369e Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 11 Jun 2019 17:12:09 +0200 Subject: [PATCH 10/27] Let's keep it undocumented for now --- ecological/config.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 73bbcbc..01c57a5 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -2,8 +2,7 @@ import enum import os import warnings -from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, - get_type_hints) +from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints from . import casting @@ -29,7 +28,7 @@ class Autoload(enum.Enum): NEVER = "NEVER" -def environ_name(attr_name: str, prefix: Optional[str] = None): +def _generate_environ_name(attr_name: str, prefix: Optional[str] = None): variable_name = "" if prefix: variable_name += f"{prefix}_" @@ -38,10 +37,6 @@ def environ_name(attr_name: str, prefix: Optional[str] = None): return variable_name.upper() -def environb_name(*args, **kwargs): - return environ_name(*args, **kwargs).encode() - - @dataclasses.dataclass class _Options: """ @@ -54,7 +49,7 @@ class _Options: source: Source = os.environ transform: TransformCallable = casting.cast wanted_type: Type = str - variable_name: Callable[[str, Optional[str]], VariableName] = environ_name + variable_name: Callable[[str, Optional[str]], VariableName] = _generate_environ_name @classmethod def from_dict(cls, options_dict: Dict) -> "_Options": From 709c815cfa35712c4d1880ce217f4aefdd10c54d Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Wed, 12 Jun 2019 20:39:08 +0200 Subject: [PATCH 11/27] Apply suggestions from code review Co-Authored-By: Thibaut Le Page --- ecological/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 01c57a5..a072100 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -101,14 +101,14 @@ def get(self) -> VariableValue: if self.default is _NO_DEFAULT: raise AttributeError( f"Configuration error: '{self.variable_name}' is not set." - ) from e + ) from None else: return self.default try: return self.transform(raw_value, self.wanted_type) except (ValueError, SyntaxError) as e: - raise ValueError(f"Invalid configuration for '{self.variable_name}': {e}.") + raise ValueError(f"Invalid configuration for '{self.variable_name}': {e}.") from e class Config: From e5038f57bc0a550b2d3714dd0c1205432724ae9b Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Wed, 12 Jun 2019 21:01:45 +0200 Subject: [PATCH 12/27] Correct exceptions causes --- ecological/config.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index a072100..e57d323 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -2,7 +2,8 @@ import enum import os import warnings -from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints +from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, + get_type_hints) from . import casting @@ -28,7 +29,9 @@ class Autoload(enum.Enum): NEVER = "NEVER" -def _generate_environ_name(attr_name: str, prefix: Optional[str] = None): +def _generate_environ_name( + attr_name: str, prefix: Optional[str] = None +) -> VariableName: variable_name = "" if prefix: variable_name += f"{prefix}_" @@ -97,7 +100,7 @@ def set_defaults( def get(self) -> VariableValue: try: raw_value = self.source[self.variable_name] - except KeyError as e: + except KeyError: if self.default is _NO_DEFAULT: raise AttributeError( f"Configuration error: '{self.variable_name}' is not set." @@ -108,7 +111,9 @@ def get(self) -> VariableValue: try: return self.transform(raw_value, self.wanted_type) except (ValueError, SyntaxError) as e: - raise ValueError(f"Invalid configuration for '{self.variable_name}': {e}.") from e + raise ValueError( + f"Invalid configuration for '{self.variable_name}'." + ) from e class Config: From 6352c33f46725354d012763b82ddfaf78bac061b Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Wed, 12 Jun 2019 21:41:39 +0200 Subject: [PATCH 13/27] Documentation effort. God forgive my written English --- ecological/casting.py | 4 ++++ ecological/config.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ecological/casting.py b/ecological/casting.py index 5156edc..f5e74d4 100644 --- a/ecological/casting.py +++ b/ecological/casting.py @@ -1,3 +1,7 @@ +""" +Provides ``cast`` function (default transform function for variables of `config.Config`) +mitigating a number of `typing` module quirks that happen across Python versions. +""" import ast import collections from typing import AnyStr, ByteString, Dict, FrozenSet, List, Set, Tuple diff --git a/ecological/config.py b/ecological/config.py index e57d323..4bf97f7 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -1,9 +1,12 @@ +""" +The heart of the library. +See ``README.rst`` and ``Configuration`` class for more details. +""" import dataclasses import enum import os import warnings -from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, - get_type_hints) +from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints from . import casting @@ -21,7 +24,7 @@ class Autoload(enum.Enum): - ``Autoload.CLASS`` - load variable values to class on its subclass creation, - ``Autoload.OBJECT`` - load variable values to object instance on its initialization, - - ``Autoload.NEVER`` - does not perform any autoloading; ``Config.load`` method needs to called explicitly. + - ``Autoload.NEVER`` - does not perform any autoloading; ``Config.load`` method needs to be called explicitly. """ CLASS = "CLASS" @@ -32,6 +35,13 @@ class Autoload(enum.Enum): def _generate_environ_name( attr_name: str, prefix: Optional[str] = None ) -> VariableName: + """ + Outputs an environment variable name based on the ``Configuration``'s + subclass attribute name and the optional prefix. + + >>> _generate_environ_name("attr_name", prefix="prefixed") + "PREFIXED_ATTR_NAME" + """ variable_name = "" if prefix: variable_name += f"{prefix}_" @@ -78,6 +88,11 @@ def from_dict(cls, options_dict: Dict) -> "_Options": @dataclasses.dataclass class Variable: + """ + Represents a single variable from the configuration source + and user preferences how to process it. + """ + variable_name: Optional[VariableName] = None default: Any = _NO_DEFAULT transform: Optional[TransformCallable] = None @@ -92,12 +107,21 @@ def set_defaults( source: Source, wanted_type: Type, ): + """ + Sets missing properties of the instance of `Variable`` in order to + be able to fetch its value with the ``Variable.get`` method. + """ self.variable_name = self.variable_name or variable_name self.transform = self.transform or transform self.source = self.source or source self.wanted_type = wanted_type def get(self) -> VariableValue: + """ + Fetches a value of variable from the ``self.source`` and invoke the + ``self.transform`` operation on it. Falls back to ``self.default`` + if the value is not found. + """ try: raw_value = self.source[self.variable_name] except KeyError: @@ -207,8 +231,11 @@ def load(cls: "Config", target_obj: Optional[object] = None): setattr(target_obj, attr_name, variable.get()) -# DEPRECATED: For backward compatibility purposes only class AutoConfig(Config, autoload=Autoload.NEVER): + """ + DEPRECATED: For backward compatibility purposes only; please use ``ecological.Config`` instead. + """ + def __init_subclass__(cls, prefix: Optional[str] = None, **kwargs): warnings.warn( "ecological.AutoConfig is deprecated, please use ecological.Config instead.", From d6eb21beadc63813c0ca370d127ac856121ae6b1 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Wed, 12 Jun 2019 22:33:43 +0200 Subject: [PATCH 14/27] More documentation --- README.rst | 118 +++++++++++++++++++++---------------------- ecological/config.py | 3 +- poetry.lock | 79 ++++++++++++++++++++++++++++- pyproject.toml | 2 + 4 files changed, 141 insertions(+), 61 deletions(-) diff --git a/README.rst b/README.rst index 4c4fdd5..4decddc 100644 --- a/README.rst +++ b/README.rst @@ -35,52 +35,6 @@ class properties from the environment variables with the same (but upper cased) By default the values are set at the class definition type and assigned to the class itself (i.e. the class doesn't need to be instantiated). If needed this behavior can be changed (see the next section). -Autoloading -============= -It is possible to defer/disable autoloading (setting) of variable values by specifying the ``autoload`` option on class definition. - -On class creation (default) ---------------------------- -When no option is provided values are loaded immediately on class creation and assigned to class attributes: - -.. code-block:: python - - class Configuration(ecological.Config): - port: int - # Values already read and set at this point. - # assert Configuration.port == - -Never ------- -When this option is chosen, no autoloading happens. In order to set variable values, the ``Config.load`` method needs to be called explicitly: - -.. code-block:: python - - class Configuration(ecological.Config, autoload=ecological.Autoload.NEVER): - port: int - # Values not set at this point. - # Accessing Configuration.port would throw AttributeError. - - Configuration.load() - # Values read and set at this point. - # assert Configuration.port == - -On object instance initialization ----------------------------------- -If it is preferred to load and store attribute values on the object instance instead of the class itself, the ``Autoload.OBJECT`` strategy can be used: - -.. code-block:: python - - class Configuration(ecological.Config, autoload=ecological.Autoload.OBJECT): - port: int - # Values not set at this point. - - config = Configuration() - # Values read and set at this point on ``config``. - # assert config.port == - # Accessing ``Configuration.port`` would throw AttributeError. - - Typing Support ============== ``Ecological`` also supports some of the types defined in PEP484_, for example: @@ -108,39 +62,85 @@ You can also decide to prefix your application configuration, for example, to av In this case the ``home`` property will be fetched from the ``MYAPP_HOME`` environment property. -Fine-grained control --------------------- +Nested Configuration +===================== +``Ecological.Config`` also supports nested configurations, for example: + +.. code-block:: python + + + class Configuration(ecological.Config): + integer: int + + class Nested(ecological.Config, prefix='nested'): + boolean: bool + +This way you can group related configuration properties hierarchically. + +Fine-grained Control +==================== You can control how the configuration properties are set by providing a ``ecological.Variable`` instance as the default value. ``ecological.Variable`` receives the following parameters: -- ``variable_name`` (mandatory) - exact name of the environment variable that will be used. +- ``variable_name`` (optional) - exact name of the environment variable that will be used. If not provided + the default mechanism of deriving a name from the attribute name will be applied. - ``default`` (optional) - default value for the property if it isn't set. - ``transform`` (optional) - function that converts the string in the environment to the value and type you expect in your application. The default ``transform`` function will try to cast the string to the annotation type of the property. +- ``source`` (optional) - dictionary that the value will be loaded from; defaults to ``os.environ``. Transformation function -....................... - +----------------------- The transformation function receive two parameters, a string ``representation`` with the raw value, and a ``wanted_type`` with the value of the annotation (usually, but not necessarily a ``type``). -Nested Configuration --------------------- -``Ecological.Config`` also supports nested configurations, for example: +Autoloading +=========== +It is possible to defer/disable autoloading (setting) of variable values by specifying the ``autoload`` option on class definition. -.. code-block:: python +On class creation (default) +--------------------------- +When no option is provided values are loaded immediately on class creation and assigned to class attributes: +.. code-block:: python class Configuration(ecological.Config): - integer: int + port: int + # Values already read and set at this point. + # assert Configuration.port == - class Nested(ecological.Config, prefix='nested'): - boolean: bool +Never +------ +When this option is chosen, no autoloading happens. In order to set variable values, the ``Config.load`` method needs to be called explicitly: -This way you can group related configuration properties hierarchically. +.. code-block:: python + + class Configuration(ecological.Config, autoload=ecological.Autoload.NEVER): + port: int + # Values not set at this point. + # Accessing Configuration.port would throw AttributeError. + + Configuration.load() + # Values read and set at this point. + # assert Configuration.port == + +On object instance initialization +---------------------------------- +If it is preferred to load and store attribute values on the object instance instead of the class itself, the ``Autoload.OBJECT`` strategy can be used: + +.. code-block:: python + + class Configuration(ecological.Config, autoload=ecological.Autoload.OBJECT): + port: int + # Values not set at this point. + + config = Configuration() + # Values read and set at this point on ``config``. + # assert config.port == + # Accessing ``Configuration.port`` would throw AttributeError. Tutorial ======== diff --git a/ecological/config.py b/ecological/config.py index 4bf97f7..8db4d40 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -6,7 +6,8 @@ import enum import os import warnings -from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints +from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, + get_type_hints) from . import casting diff --git a/poetry.lock b/poetry.lock index 1fa20b5..0e75eca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -36,6 +36,14 @@ attrs = ">=17.4.0" click = ">=6.5" toml = ">=0.9.4" +[[package]] +category = "dev" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + [[package]] category = "dev" description = "Composable command line interface toolkit" @@ -69,6 +77,29 @@ optional = false python-versions = "*" version = "0.6" +[[package]] +category = "dev" +description = "Style checker for Sphinx (or other) RST documentation" +name = "doc8" +optional = false +python-versions = "*" +version = "0.8.0" + +[package.dependencies] +chardet = "*" +docutils = "*" +restructuredtext-lint = ">=0.7" +six = "*" +stevedore = "*" + +[[package]] +category = "dev" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" +optional = false +python-versions = "*" +version = "0.14" + [[package]] category = "dev" description = "Discover and load entry points from installed packages." @@ -139,6 +170,14 @@ version = "19.0" pyparsing = ">=2.0.2" six = "*" +[[package]] +category = "dev" +description = "Python Build Reasonableness" +name = "pbr" +optional = false +python-versions = "*" +version = "5.2.1" + [[package]] category = "dev" description = "plugin and hook calling mechanisms for python" @@ -174,6 +213,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.1.1" +[[package]] +category = "dev" +description = "Pygments is a syntax highlighting package written in Python." +name = "pygments" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.4.2" + [[package]] category = "dev" description = "Python parsing module" @@ -217,6 +264,17 @@ version = "2.7.1" coverage = ">=4.4" pytest = ">=3.6" +[[package]] +category = "dev" +description = "reStructuredText linter" +name = "restructuredtext-lint" +optional = false +python-versions = "*" +version = "1.3.0" + +[package.dependencies] +docutils = ">=0.11,<1.0" + [[package]] category = "dev" description = "a python refactoring library..." @@ -233,6 +291,18 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*" version = "1.12.0" +[[package]] +category = "dev" +description = "Manage dynamic plugins for Python applications" +name = "stevedore" +optional = false +python-versions = "*" +version = "1.30.1" + +[package.dependencies] +pbr = ">=2.0.0,<2.1.0 || >2.1.0" +six = ">=1.10.0" + [[package]] category = "dev" description = "Python Library for Tom's Obvious, Minimal Language" @@ -283,7 +353,7 @@ python-versions = ">=2.7" version = "0.5.1" [metadata] -content-hash = "e7c8c6f2ef25b1b836c079e8d1edc7759c8e9c699764163fd81e0369b1a3a235" +content-hash = "a459a098ec36db016ed6f1983da3aa26cfb0f48b39a943d8a3640dcab1f8efe6" python-versions = "^3.6" [metadata.hashes] @@ -291,10 +361,13 @@ appdirs = ["9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", " atomicwrites = ["03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"] attrs = ["69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", "f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"] black = ["817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", "e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5"] +chardet = ["84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"] click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"] colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"] coverage = ["007eeef7e23f9473622f7d94a3e029a45d55a92a1f083f0f3512f5ab9a669b05", "0388c12539372bb92d6dde68b4627f0300d948965bbb7fc104924d715fdc0965", "079248312838c4c8f3494934ab7382a42d42d5f365f0cf7516f938dbb3f53f3f", "17307429935f96c986a1b1674f78079528833410750321d22b5fb35d1883828e", "1afccd7e27cac1b9617be8c769f6d8a6d363699c9b86820f40c74cfb3328921c", "2ad357d12971e77360034c1596011a03f50c0f9e1ecd12e081342b8d1aee2236", "2b4d7f03a8a6632598cbc5df15bbca9f778c43db7cf1a838f4fa2c8599a8691a", "2e1a5c6adebb93c3b175103c2f855eda957283c10cf937d791d81bef8872d6ca", "309d91bd7a35063ec7a0e4d75645488bfab3f0b66373e7722f23da7f5b0f34cc", "358d635b1fc22a425444d52f26287ae5aea9e96e254ff3c59c407426f44574f4", "3f4d0b3403d3e110d2588c275540649b1841725f5a11a7162620224155d00ba2", "43a155eb76025c61fc20c3d03b89ca28efa6f5be572ab6110b2fb68eda96bfea", "493082f104b5ca920e97a485913de254cbe351900deed72d4264571c73464cd0", "4c4f368ffe1c2e7602359c2c50233269f3abe1c48ca6b288dcd0fb1d1c679733", "5ff16548492e8a12e65ff3d55857ccd818584ed587a6c2898a9ebbe09a880674", "66f393e10dd866be267deb3feca39babba08ae13763e0fc7a1063cbe1f8e49f6", "700d7579995044dc724847560b78ac786f0ca292867447afda7727a6fbaa082e", "81912cfe276e0069dca99e1e4e6be7b06b5fc8342641c6b472cb2fed7de7ae18", "82cbd3317320aa63c65555aa4894bf33a13fb3a77f079059eb5935eea415938d", "845fddf89dca1e94abe168760a38271abfc2e31863fbb4ada7f9a99337d7c3dc", "87d942863fe74b1c3be83a045996addf1639218c2cb89c5da18c06c0fe3917ea", "9721f1b7275d3112dc7ccf63f0553c769f09b5c25a26ee45872c7f5c09edf6c1", "a4497faa4f1c0fc365ba05eaecfb6b5d24e3c8c72e95938f9524e29dadb15e76", "a7cfaebd8f24c2b537fa6a271229b051cdac9c1734bb6f939ccfc7c055689baa", "ab3508df9a92c1d3362343d235420d08e2662969b83134f8a97dc1451cbe5e84", "b0059630ca5c6b297690a6bf57bf2fdac1395c24b7935fd73ee64190276b743b", "b6cebae1502ce5b87d7c6f532fa90ab345cfbda62b95aeea4e431e164d498a3d", "bd4800e32b4c8d99c3a2c943f1ac430cbf80658d884123d19639bcde90dad44a", "cdd92dd9471e624cd1d8c1a2703d25f114b59b736b0f1f659a98414e535ffb3d", "d00e29b78ff610d300b2c37049a41234d48ea4f2d2581759ebcf67caaf731c31", "d1ee76f560c3c3e8faada866a07a32485445e16ed2206ac8378bd90dadffb9f0", "dd707a21332615108b736ef0b8513d3edaf12d2a7d5fc26cd04a169a8ae9b526", "e3ba9b14607c23623cf38f90b23f5bed4a3be87cbfa96e2e9f4eabb975d1e98b", "e9a0e1caed2a52f15c96507ab78a48f346c05681a49c5b003172f8073da6aa6b", "eea9135432428d3ca7ee9be86af27cb8e56243f73764a9b6c3e0bda1394916be", "f29841e865590af72c4b90d7b5b8e93fd560f5dea436c1d5ee8053788f9285de", "f3a5c6d054c531536a83521c00e5d4004f1e126e2e2556ce399bef4180fbe540", "f87f522bde5540d8a4b11df80058281ac38c44b13ce29ced1e294963dd51a8f8", "f8c55dd0f56d3d618dfacf129e010cbe5d5f94b6951c1b2f13ab1a2f79c284da", "f98b461cb59f117887aa634a66022c0bd394278245ed51189f63a036516e32de"] dataclasses = ["454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f", "6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"] +doc8 = ["2df89f9c1a5abfb98ab55d0175fed633cae0cf45025b8b1e0ee5ea772be28543", "d12f08aa77a4a65eb28752f4bc78f41f611f9412c4155e2b03f1f5d4a45efe04"] +docutils = ["02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", "51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", "7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"] entrypoints = ["589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", "c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"] filelock = ["18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", "929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"] flake8 = ["859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", "a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"] @@ -302,15 +375,19 @@ importlib-metadata = ["a9f185022cfa69e9ca5f7eabfd5a58b689894cb78a11e3c8c89398a8c mccabe = ["ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", "dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"] more-itertools = ["2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", "c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"] packaging = ["0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", "9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"] +pbr = ["0ce920b865091450bbcd452b35cf6d6eb8a6d9ce13ad2210d6e77557f85cf32b", "93d2dc6ee0c9af4dbc70bc1251d0e545a9910ca8863774761f92716dece400b6"] pluggy = ["0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", "b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"] py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"] pycodestyle = ["95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", "e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"] pyflakes = ["17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", "d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"] +pygments = ["71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", "881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"] pyparsing = ["1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", "9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"] pytest = ["6032845e68a17a96e8da3088037f899b56357769a724122056265ca2ea1890ee", "bea27a646a3d74cbbcf8d3d4a06b2dfc336baf3dc2cc85cf70ad0157e73e8322"] pytest-cov = ["2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6", "e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a"] +restructuredtext-lint = ["97b3da356d5b3a8514d8f1f9098febd8b41463bed6a1d9f126cf0a048b6fd908"] rope = ["6b728fdc3e98a83446c27a91fc5d56808a004f8beab7a31ab1d7224cecc7d969", "c5c5a6a87f7b1a2095fb311135e2a3d1f194f5ecb96900fdd0a9100881f48aaf", "f0dcf719b63200d492b85535ebe5ea9b29e0d0b8aebeb87fe03fc1a65924fdaf"] six = ["3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"] +stevedore = ["7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0", "7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2"] toml = ["229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", "235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", "f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"] tox = ["f5c8e446b51edd2ea97df31d4ded8c8b72e7d6c619519da6bb6084b9dd5770f9", "f87fd33892a2df0950e5e034def9468988b8d008c7e9416be665fcc0dd45b14f"] virtualenv = ["99acaf1e35c7ccf9763db9ba2accbca2f4254d61d1912c5ee364f9cc4a8942a0", "fe51cdbf04e5d8152af06c075404745a7419de27495a83f0d72518ad50be3ce8"] diff --git a/pyproject.toml b/pyproject.toml index f33b84b..d988ddd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ pytest-cov = "^2.7" black = {version = "^18.3-alpha.0",allows-prereleases = true} rope = "^0.14.0" flake8 = "^3.7" +doc8 = "^0.8.0" +pygments = "^2.4" [build-system] requires = ["poetry>=0.12"] From 7a7c5fc77fb9f03e3d21ac40009c8261255405b2 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Thu, 13 Jun 2019 11:01:44 +0200 Subject: [PATCH 15/27] Config.load docs --- ecological/config.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 8db4d40..26f09a0 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -187,30 +187,21 @@ def __init__(self, *args, **kwargs): @classmethod def load(cls: "Config", target_obj: Optional[object] = None): """ - The class' ``attribute_dict`` includes attributes with a default value and some special - keys like ``__annotations__`` which includes annotations of all attributes, including the - ones that don't have a value. - - To simplify the class building process the annotated attributes that don't have a value - in the ``attribute_dict`` are injected with a ``_NO_DEFAULT`` sentinel object. - - After this all the public attributes of the class are iterated over and one - of three things is performed: - - - If the attribute value is an instance of ``Config`` it is kept as is to allow nested - configuration. - - If the attribute value is of type ``Variable``, ``Config`` will call its ``get`` - method with the attribute's annotation type as the only parameter - - Otherwise, ``Config`` will create a ``Variable`` instance, with - "{prefix}_{attribute_name}" as the environment variable name and the attribute value - (the default value or ``_NO_DEFAULT``) and do the same process as in the previous point. + Fetches and converts values of variables declared as attributes on ``cls`` according + to their specification and finally assigns them to the corresponding attributes + on ``target_obj`` (which by default is ``cls`` itself). """ target_obj = target_obj or cls cls_dict = vars(cls).copy() attr_types: Dict[str, type] = get_type_hints(cls) + # There is no single place that has all class attributes regardless of + # having default value or not. Thus keys of cls.__annotations__ and + # cls.__dict__ are merged providing a complete list. attr_names = set(cls_dict).union(attr_types.keys()) for attr_name in attr_names: + # Omit private attributes and nested configuration + # (Attribute value can be the instance of Config itself). if attr_name.startswith("_") or isinstance(attr_name, cls): continue attr_value = cls_dict.get(attr_name, _NO_DEFAULT) From 80a6b716c9f9df660c3ca980cfc9039b494e3659 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Thu, 13 Jun 2019 11:03:56 +0200 Subject: [PATCH 16/27] Language! --- ecological/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecological/config.py b/ecological/config.py index 26f09a0..5b0adbb 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -195,12 +195,12 @@ def load(cls: "Config", target_obj: Optional[object] = None): cls_dict = vars(cls).copy() attr_types: Dict[str, type] = get_type_hints(cls) # There is no single place that has all class attributes regardless of - # having default value or not. Thus keys of cls.__annotations__ and + # having default value or not. Thus the keys of cls.__annotations__ and # cls.__dict__ are merged providing a complete list. attr_names = set(cls_dict).union(attr_types.keys()) for attr_name in attr_names: - # Omit private attributes and nested configuration + # Omit private and nested configuration attributes # (Attribute value can be the instance of Config itself). if attr_name.startswith("_") or isinstance(attr_name, cls): continue From 5714bcb74388370b726f473f75ecfd5c3662f88b Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Thu, 13 Jun 2019 16:36:14 +0200 Subject: [PATCH 17/27] Parameters uber-table WIP --- README.rst | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 4decddc..abf9b28 100644 --- a/README.rst +++ b/README.rst @@ -79,23 +79,41 @@ This way you can group related configuration properties hierarchically. Fine-grained Control ==================== -You can control how the configuration properties are set by providing a ``ecological.Variable`` instance as the default -value. -``ecological.Variable`` receives the following parameters: ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| Option | Class level | Variable level | Default | Description | ++===================+===============+=================+=================================================+=================================================================+ +| ``prefix`` | yes | no | ``None`` | Prefix that is prepended when a variable name is derived from | +| | | | | an attribute name. | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| ``variable_name`` | yes | yes | Derived from attribute name and prefixed | When specified on the variable level it states | +| | | | with ``prefix`` if specified; uppercased. | exact name of the source variable that will be used. | +| | | | | | +| | | | | When specified on the class level it is treated as a function | +| | | | | that returns a variable name from an attribute name with | +| | | | | the following signature: | +| | | | | | +| | | | | ``def func(attribute_name: str, prefix: Optional[str] = None)`` | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| ``default`` | no | yes | (no default) | Default value for the property if it isn't set. | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| ``transform`` | yes | yes | A source value is casted to the ``wanted_type`` | Function that converts a value from the ``source`` to the value | +| | | | (``ecological.casting.cast``). | and ``wanted_type`` you expect with the following signature: | +| | | | | | +| | | | | ``def func(source_value: str, wanted_type: Union[Type, str])`` | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| ``source`` | yes | yes | ``os.environ`` | Dictionary that the value will be loaded from. | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ +| ``wanted_type`` | yes | yes | ``str`` | | ++-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -- ``variable_name`` (optional) - exact name of the environment variable that will be used. If not provided - the default mechanism of deriving a name from the attribute name will be applied. -- ``default`` (optional) - default value for the property if it isn't set. -- ``transform`` (optional) - function that converts the string in the environment to the value and type you - expect in your application. The default ``transform`` function will try to cast the string to the annotation - type of the property. -- ``source`` (optional) - dictionary that the value will be loaded from; defaults to ``os.environ``. +You can control how the configuration properties are set by providing a ``ecological.Variable`` instance as the default +value. -Transformation function ------------------------ -The transformation function receive two parameters, a string ``representation`` with the raw value, and a -``wanted_type`` with the value of the annotation (usually, but not necessarily a ``type``). +Class level options +-------------------- +Some of the parameters can also be specified on the class level in order to avoid repetition +or unnecessary ``ecological.Variable`` declarations when only a subset of options is to be changed globally: Autoloading =========== From 30c7ea4257bd18d49e8ecf9e286f54396d83f2de Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:03:52 +0200 Subject: [PATCH 18/27] Table almost final --- README.rst | 82 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index abf9b28..2d1b063 100644 --- a/README.rst +++ b/README.rst @@ -79,41 +79,57 @@ This way you can group related configuration properties hierarchically. Fine-grained Control ==================== - -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| Option | Class level | Variable level | Default | Description | -+===================+===============+=================+=================================================+=================================================================+ -| ``prefix`` | yes | no | ``None`` | Prefix that is prepended when a variable name is derived from | -| | | | | an attribute name. | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| ``variable_name`` | yes | yes | Derived from attribute name and prefixed | When specified on the variable level it states | -| | | | with ``prefix`` if specified; uppercased. | exact name of the source variable that will be used. | -| | | | | | -| | | | | When specified on the class level it is treated as a function | -| | | | | that returns a variable name from an attribute name with | -| | | | | the following signature: | -| | | | | | -| | | | | ``def func(attribute_name: str, prefix: Optional[str] = None)`` | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| ``default`` | no | yes | (no default) | Default value for the property if it isn't set. | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| ``transform`` | yes | yes | A source value is casted to the ``wanted_type`` | Function that converts a value from the ``source`` to the value | -| | | | (``ecological.casting.cast``). | and ``wanted_type`` you expect with the following signature: | -| | | | | | -| | | | | ``def func(source_value: str, wanted_type: Union[Type, str])`` | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| ``source`` | yes | yes | ``os.environ`` | Dictionary that the value will be loaded from. | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ -| ``wanted_type`` | yes | yes | ``str`` | | -+-------------------+---------------+-----------------+-------------------------------------------------+-----------------------------------------------------------------+ - You can control how the configuration properties are set by providing a ``ecological.Variable`` instance as the default -value. +value for an attribute or by specifying global options on the class level: + +.. code-block:: python -Class level options --------------------- -Some of the parameters can also be specified on the class level in order to avoid repetition -or unnecessary ``ecological.Variable`` declarations when only a subset of options is to be changed globally: + my_source: Dict = {"KEY1": "VALUE1"} + + class Configuration(ecological.Config, transform=lambda v, wt: v, wanted_type=int, ...): + my_var1: WantedType = ecological.Variable(transform=lambda v, wt: wt(v), source=my_source, ...) + my_var2: str + # ... + +All possible options and their meaning can be found in the table below: + ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| Option | Class level | Variable level | Default | Description | ++===================+===============+=================+=================================================+===================================================================+ +| ``prefix`` | yes | no | ``None`` | A prefix that is prepended when a variable name is derived from | +| | | | | an attribute name. | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| ``variable_name`` | yes | yes | Derived from attribute name and prefixed | When specified on the variable level it states | +| | | | with ``prefix`` if specified; uppercased. | the exact name of the source variable that will be used. | +| | | | | | +| | | | | When specified on the class level it is treated as a function | +| | | | | that returns a variable name from the attribute name with | +| | | | | the following signature: | +| | | | | | +| | | | | ``def func(attribute_name: str, prefix: Optional[str] = None)`` | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| ``default`` | no | yes | (no default) | Default value for the property if it isn't set. | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| ``transform`` | yes | yes | A source value is casted to the ``wanted_type`` | A function that converts a value from the ``source`` to the value | +| | | | (``ecological.casting.cast``). | and ``wanted_type`` you expect with the following signature: | +| | | | | | +| | | | | ``def func(source_value: str, wanted_type: Union[Type, str])`` | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| ``source`` | yes | yes | ``os.environ`` | Dictionary that the value will be loaded from. | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ +| ``wanted_type`` | yes | yes | ``str`` | Desired Python type of the attribute's value. | +| | | | | | +| | | | | On the variable level it is specified via a type annotation on | +| | | | | the attribute: ``my_var_1: my_wanted_type``. | +| | | | | | +| | | | | However it can be also specified on the class level, then it acts | +| | | | | as a default when the annotation is not provided: | +| | | | | | +| | | | | ``class MyConfig(ecological.Config, wanted_type=int, ...)`` | ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ + +.. note:: Please mind that in the case of specyfing options on both levels (variable and class) + the variable ones take precedence over class ones. Autoloading =========== From c57d92a2667244d2333bc04dfbe9d9ee87d4f00d Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:25:17 +0200 Subject: [PATCH 19/27] Add comments to finish the table --- README.rst | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 2d1b063..4661789 100644 --- a/README.rst +++ b/README.rst @@ -77,9 +77,14 @@ Nested Configuration This way you can group related configuration properties hierarchically. +Advanced +======== + Fine-grained Control -==================== -You can control how the configuration properties are set by providing a ``ecological.Variable`` instance as the default +--------------------- +You can control some behavior of how the configuration properties are set. + +It can be achieved by providing a ``ecological.Variable`` instance as the default value for an attribute or by specifying global options on the class level: .. code-block:: python @@ -128,15 +133,21 @@ All possible options and their meaning can be found in the table below: | | | | | ``class MyConfig(ecological.Config, wanted_type=int, ...)`` | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ -.. note:: Please mind that in the case of specyfing options on both levels (variable and class) - the variable ones take precedence over class ones. +Following rules apply when options are resolved: + +- in the case of specyfing options on both levels (variable and class) + the variable ones take precedence over class ones, +- when some options are missing on the variable level their defaults + are taken from the class level, +- it is not necessary to assign an ``ecological.Variable`` instance to + change the behavior; it can still be changed on the class level (globally). Autoloading -=========== +------------ It is possible to defer/disable autoloading (setting) of variable values by specifying the ``autoload`` option on class definition. On class creation (default) ---------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~ When no option is provided values are loaded immediately on class creation and assigned to class attributes: .. code-block:: python @@ -147,7 +158,7 @@ When no option is provided values are loaded immediately on class creation and a # assert Configuration.port == Never ------- +~~~~~ When this option is chosen, no autoloading happens. In order to set variable values, the ``Config.load`` method needs to be called explicitly: .. code-block:: python @@ -162,7 +173,7 @@ When this option is chosen, no autoloading happens. In order to set variable val # assert Configuration.port == On object instance initialization ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If it is preferred to load and store attribute values on the object instance instead of the class itself, the ``Autoload.OBJECT`` strategy can be used: .. code-block:: python From 5178428c9ba43aad79f8b5b3ffc8b9cacbd72d87 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:31:37 +0200 Subject: [PATCH 20/27] Tutorial section more to the top --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 4661789..91d4234 100644 --- a/README.rst +++ b/README.rst @@ -77,6 +77,11 @@ Nested Configuration This way you can group related configuration properties hierarchically. +Tutorial +======== +The `tutorial `_ includes real examples of all the available +features. + Advanced ======== @@ -187,11 +192,6 @@ If it is preferred to load and store attribute values on the object instance ins # assert config.port == # Accessing ``Configuration.port`` would throw AttributeError. -Tutorial -======== -The `tutorial `_ includes real examples of all the available -features. - Caveats and Known Limitations ============================= From 26eaf0eb0eca78bb2b52c520af58d7646c28aea6 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:39:45 +0200 Subject: [PATCH 21/27] Rephrase tutorial section --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 91d4234..55d2009 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,10 @@ class properties from the environment variables with the same (but upper cased) By default the values are set at the class definition type and assigned to the class itself (i.e. the class doesn't need to be instantiated). If needed this behavior can be changed (see the next section). +Tutorial +-------- +The `tutorial `_ can be used to get to know with the library's basic features interactively. + Typing Support ============== ``Ecological`` also supports some of the types defined in PEP484_, for example: @@ -77,11 +81,6 @@ Nested Configuration This way you can group related configuration properties hierarchically. -Tutorial -======== -The `tutorial `_ includes real examples of all the available -features. - Advanced ======== From 170a3a60af7e05cea522a8bf232b5276a03a82ea Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:42:23 +0200 Subject: [PATCH 22/27] Tutorial more up to date --- tutorial.ipynb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index 80f5e55..7377e4a 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -44,7 +44,7 @@ "source": [ "import ecological\n", "\n", - "class Configuration(ecological.AutoConfig):\n", + "class Configuration(ecological.Config):\n", " integer_list: list\n", " integer: int\n", " dictionary: dict\n", @@ -201,7 +201,7 @@ "source": [ "from typing import List, Dict\n", "\n", - "class ConfigurationTyping(ecological.AutoConfig):\n", + "class ConfigurationTyping(ecological.Config):\n", " integer_list: List\n", " dictionary: Dict" ] @@ -275,7 +275,7 @@ "os.environ[\"CONFIG_HOME\"] = \"/app/home\"\n", "os.environ[\"CONFIG_VALUE\"] = \"Prefixed\"\n", "\n", - "class ConfigurationPrefix(ecological.AutoConfig, prefix=\"config\"):\n", + "class ConfigurationPrefix(ecological.Config, prefix=\"config\"):\n", " home: str\n", " value: str" ] @@ -331,7 +331,7 @@ "\n", "`ecological.Variable` receives the following parameters:\n", "\n", - "`variable_name` (mandatory) - exact name of the environment variable that will be used.\n", + "`variable_name` (optional) - exact name of the environment variable that will be used.\n", "`default` (optional) - default value for the property if it isn't set.\n", "`transform` (optional) - function that converts the string in the environment to the value and type you expect in your application. The default `transform` function will try to cast the string to the annotation type of the property.\n", "\n", @@ -354,7 +354,7 @@ " assert wanted_type is int\n", " return int(value) * 2\n", "\n", - "class ConfigurationVariable(ecological.AutoConfig, prefix=\"this_is_going_to_be_ignored\"):\n", + "class ConfigurationVariable(ecological.Config, prefix=\"this_is_going_to_be_ignored\"):\n", " integer = ecological.Variable(\"Integer\", transform=lambda v, wt: int(v))\n", " integer_x2: int = ecological.Variable(\"Integer\", transform=times_2)\n", " integer_as_str: str = ecological.Variable(\"Integer\", transform=lambda v, wt: v)\n", @@ -449,7 +449,7 @@ "source": [ "## Nested Configuration\n", "\n", - "`Ecological.AutoConfig` also supports nested configurations, for example:" + "`ecological.Config` also supports nested configurations, for example:" ] }, { @@ -463,10 +463,10 @@ "os.environ[\"INTEGER\"] = \"42\"\n", "os.environ[\"NESTED_BOOLEAN\"] = \"True\"\n", "\n", - "class ConfigurationNested(ecological.AutoConfig):\n", + "class ConfigurationNested(ecological.Config):\n", " integer: int\n", "\n", - " class Nested(ecological.AutoConfig, prefix='nested'):\n", + " class Nested(ecological.Config, prefix='nested'):\n", " boolean: bool" ] }, From d7935c70683f266f2412e6cff205a9b69375c4f2 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 13:44:29 +0200 Subject: [PATCH 23/27] If casting is exposed it should be importable from top-level --- README.rst | 2 +- ecological/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 55d2009..78b98e1 100644 --- a/README.rst +++ b/README.rst @@ -120,7 +120,7 @@ All possible options and their meaning can be found in the table below: | ``default`` | no | yes | (no default) | Default value for the property if it isn't set. | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ | ``transform`` | yes | yes | A source value is casted to the ``wanted_type`` | A function that converts a value from the ``source`` to the value | -| | | | (``ecological.casting.cast``). | and ``wanted_type`` you expect with the following signature: | +| | | | (``ecological.cast``). | and ``wanted_type`` you expect with the following signature: | | | | | | | | | | | | ``def func(source_value: str, wanted_type: Union[Type, str])`` | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ diff --git a/ecological/__init__.py b/ecological/__init__.py index 9e95993..7b30316 100644 --- a/ecological/__init__.py +++ b/ecological/__init__.py @@ -1 +1,2 @@ +from .casting import cast from .config import AutoConfig, Autoload, Config, Variable From ff4993365d539e4c4668dde995c23d604cbdf887 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 14:37:33 +0200 Subject: [PATCH 24/27] Phrasing + keep private what's private --- README.rst | 4 ++-- ecological/__init__.py | 1 - ecological/config.py | 8 ++++---- ecological/{casting.py => transform.py} | 0 4 files changed, 6 insertions(+), 7 deletions(-) rename ecological/{casting.py => transform.py} (100%) diff --git a/README.rst b/README.rst index 78b98e1..5a212d5 100644 --- a/README.rst +++ b/README.rst @@ -120,8 +120,8 @@ All possible options and their meaning can be found in the table below: | ``default`` | no | yes | (no default) | Default value for the property if it isn't set. | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ | ``transform`` | yes | yes | A source value is casted to the ``wanted_type`` | A function that converts a value from the ``source`` to the value | -| | | | (``ecological.cast``). | and ``wanted_type`` you expect with the following signature: | -| | | | | | +| | | | In case of non-scalar types (+ scalar ``bool``) | and ``wanted_type`` you expect with the following signature: | +| | | | the value is Python-parsed first. | | | | | | | ``def func(source_value: str, wanted_type: Union[Type, str])`` | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ | ``source`` | yes | yes | ``os.environ`` | Dictionary that the value will be loaded from. | diff --git a/ecological/__init__.py b/ecological/__init__.py index 7b30316..9e95993 100644 --- a/ecological/__init__.py +++ b/ecological/__init__.py @@ -1,2 +1 @@ -from .casting import cast from .config import AutoConfig, Autoload, Config, Variable diff --git a/ecological/config.py b/ecological/config.py index 5b0adbb..0c1a1fb 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -6,10 +6,10 @@ import enum import os import warnings -from typing import (Any, Callable, Dict, NewType, Optional, Type, Union, - get_type_hints) +from typing import Any, Callable, Dict, NewType, Optional, Type, Union, get_type_hints -from . import casting +# Aliased in order to avoid a conflict with the _Options.transform attribute. +from . import transform as transform_module _NO_DEFAULT = object() VariableName = NewType("VariableName", Union[str, bytes]) @@ -61,7 +61,7 @@ class _Options: prefix: Optional[str] = None autoload: Autoload = Autoload.CLASS source: Source = os.environ - transform: TransformCallable = casting.cast + transform: TransformCallable = transform_module.cast wanted_type: Type = str variable_name: Callable[[str, Optional[str]], VariableName] = _generate_environ_name diff --git a/ecological/casting.py b/ecological/transform.py similarity index 100% rename from ecological/casting.py rename to ecological/transform.py From 37207b27f73fbdfbf6c0b2d8d0cf3997d64be1f0 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Mon, 17 Jun 2019 14:53:52 +0200 Subject: [PATCH 25/27] Tests for documented options --- tests/test_config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 56c4bc7..08b4b19 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -244,3 +244,30 @@ class Configuration(ecological.Config, source=my_dict): assert Configuration.implicit == "a" assert Configuration.var_without_source == "b" assert Configuration.var_with_source == "c" + + +def test_global_wanted_type_option_is_used_as_default(monkeypatch): + monkeypatch.setenv("IMPLICIT_1", "1") + monkeypatch.setenv("IMPLICIT_WITH_TYPE_2", "2") + + class Configuration(ecological.Config, wanted_type=int): + implicit_1 = 99 + implicit_with_type_2: str + + assert Configuration.implicit_1 == 1 + assert Configuration.implicit_with_type_2 == "2" + + +def test_variable_name_is_calculation_is_used_as_default(monkeypatch): + monkeypatch.setenv("IMPLICIT_THIS_IS_CRAZY", "1") + monkeypatch.setenv("MY_ARBITRARY_NAME", "2") + + def my_variable_name(attr_name, prefix=None): + return (attr_name + "_THIS_IS_CRAZY").upper() + + class Configuration(ecological.Config, variable_name=my_variable_name): + implicit: str + var_with_name = ecological.Variable("MY_ARBITRARY_NAME") + + assert Configuration.implicit == "1" + assert Configuration.var_with_name == "2" From 50187209586b791f6518880d5fc565928e814726 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 18 Jun 2019 11:41:12 +0200 Subject: [PATCH 26/27] Apply no-brainer suggestions from code review Co-Authored-By: Thibaut Le Page --- README.rst | 8 ++++---- ecological/config.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 5a212d5..28612ff 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,7 @@ value for an attribute or by specifying global options on the class level: .. code-block:: python - my_source: Dict = {"KEY1": "VALUE1"} + my_source = {"KEY1": "VALUE1"} class Configuration(ecological.Config, transform=lambda v, wt: v, wanted_type=int, ...): my_var1: WantedType = ecological.Variable(transform=lambda v, wt: wt(v), source=my_source, ...) @@ -137,11 +137,11 @@ All possible options and their meaning can be found in the table below: | | | | | ``class MyConfig(ecological.Config, wanted_type=int, ...)`` | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ -Following rules apply when options are resolved: +The following rules apply when options are resolved: -- in the case of specyfing options on both levels (variable and class) +- when options are specified on both levels (variable and class), the variable ones take precedence over class ones, -- when some options are missing on the variable level their defaults +- when some options are missing on the variable level, their default values are taken from the class level, - it is not necessary to assign an ``ecological.Variable`` instance to change the behavior; it can still be changed on the class level (globally). diff --git a/ecological/config.py b/ecological/config.py index 0c1a1fb..1cdb4bd 100644 --- a/ecological/config.py +++ b/ecological/config.py @@ -128,7 +128,7 @@ def get(self) -> VariableValue: except KeyError: if self.default is _NO_DEFAULT: raise AttributeError( - f"Configuration error: '{self.variable_name}' is not set." + f"Configuration error: {self.variable_name!r} is not set." ) from None else: return self.default @@ -137,7 +137,7 @@ def get(self) -> VariableValue: return self.transform(raw_value, self.wanted_type) except (ValueError, SyntaxError) as e: raise ValueError( - f"Invalid configuration for '{self.variable_name}'." + f"Invalid configuration for {self.variable_name!r}." ) from e From 0f1105549ec7a6290a34f304b8d9affe2837e204 Mon Sep 17 00:00:00 2001 From: Marcin Zaremba Date: Tue, 18 Jun 2019 13:43:05 +0200 Subject: [PATCH 27/27] Wording change after review --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 28612ff..4670df9 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,11 @@ .. image:: https://travis-ci.org/jmcs/ecological.svg?branch=master - :target: https://travis-ci.org/jmcs/ecological + :target: https://travis-ci.org/jmcs/ecological .. image:: https://api.codacy.com/project/badge/Grade/1ff45d0e1a5a40b8ad0569e3edb0539d :alt: Codacy Badge :target: https://www.codacy.com/app/jmcs/ecological?utm_source=github.com&utm_medium=referral&utm_content=jmcs/ecological&utm_campaign=badger - -.. image:: https://api.codacy.com/project/badge/Coverage/1ff45d0e1a5a40b8ad0569e3edb0539d + +.. image:: https://api.codacy.com/project/badge/Coverage/1ff45d0e1a5a40b8ad0569e3edb0539d :target: https://www.codacy.com/app/jmcs/ecological?utm_source=github.com&utm_medium=referral&utm_content=jmcs/ecological&utm_campaign=Badge_Coverage ========== @@ -33,7 +33,7 @@ And then set the environment variables ``PORT``, ``DEBUG`` and ``LOG_LEVEL``. `` class properties from the environment variables with the same (but upper cased) name. By default the values are set at the class definition type and assigned to the class itself (i.e. the class doesn't need to be -instantiated). If needed this behavior can be changed (see the next section). +instantiated). If needed this behavior can be changed (see the Autoloading section). Tutorial -------- @@ -105,8 +105,8 @@ All possible options and their meaning can be found in the table below: +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ | Option | Class level | Variable level | Default | Description | +===================+===============+=================+=================================================+===================================================================+ -| ``prefix`` | yes | no | ``None`` | A prefix that is prepended when a variable name is derived from | -| | | | | an attribute name. | +| ``prefix`` | yes | no | ``None`` | A prefix that is uppercased and prepended when a variable name | +| | | | | is derived from an attribute name. | +-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ | ``variable_name`` | yes | yes | Derived from attribute name and prefixed | When specified on the variable level it states | | | | | with ``prefix`` if specified; uppercased. | the exact name of the source variable that will be used. | @@ -135,7 +135,7 @@ All possible options and their meaning can be found in the table below: | | | | | as a default when the annotation is not provided: | | | | | | | | | | | | ``class MyConfig(ecological.Config, wanted_type=int, ...)`` | -+-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ ++-------------------+---------------+-----------------+-------------------------------------------------+-------------------------------------------------------------------+ The following rules apply when options are resolved: