From 9c1321ca2c033353103155289b9c593bd4487bf6 Mon Sep 17 00:00:00 2001
From: deepaerial <nazarii.oleksiuk@gmail.com>
Date: Tue, 31 May 2022 21:23:36 +0200
Subject: [PATCH 1/3] Added support for custom separator character for nedsted
variables.
---
confz/confz_source.py | 2 ++
confz/loaders/env_loader.py | 4 +++-
confz/loaders/loader.py | 2 +-
docs/source/usage/sources_loaders.rst | 2 +-
tests/loaders/test_env_loader.py | 12 ++++++++++++
5 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/confz/confz_source.py b/confz/confz_source.py
index be0b85a..e555fc9 100644
--- a/confz/confz_source.py
+++ b/confz/confz_source.py
@@ -69,6 +69,8 @@ class ConfZEnvSource(ConfZSource):
file: Optional[Path] = None
"""Built in .env file loading with lower than environment precedence. Uses UTF-8
for decoding."""
+ nested_separator: str = "."
+ """Separator will be used in nested environment variables."""
@dataclass
diff --git a/confz/loaders/env_loader.py b/confz/loaders/env_loader.py
index 0374d6a..719e88d 100644
--- a/confz/loaders/env_loader.py
+++ b/confz/loaders/env_loader.py
@@ -68,5 +68,7 @@ def populate_config(cls, config: dict, confz_source: ConfZEnvSource):
env_vars[var_name] = origin_env_vars[env_var]
- env_vars = cls.transform_nested_dicts(env_vars)
+ env_vars = cls.transform_nested_dicts(
+ env_vars, separator=confz_source.nested_separator
+ )
cls.update_dict_recursively(config, env_vars)
diff --git a/confz/loaders/loader.py b/confz/loaders/loader.py
index e2334be..1c3503b 100644
--- a/confz/loaders/loader.py
+++ b/confz/loaders/loader.py
@@ -42,7 +42,7 @@ def transform_nested_dicts(
"""
dict_out: Dict[str, Any] = {}
for key, value in dict_in.items():
- if separator in key:
+ if separator in key and not key.startswith(separator):
inner_keys = key.split(separator)
dict_inner = dict_out
for idx, inner_key in enumerate(inner_keys):
diff --git a/docs/source/usage/sources_loaders.rst b/docs/source/usage/sources_loaders.rst
index 1570b1c..e70202d 100644
--- a/docs/source/usage/sources_loaders.rst
+++ b/docs/source/usage/sources_loaders.rst
@@ -15,7 +15,7 @@ There are multiple config sources which support a heterogeneous set of use-cases
The filename can either be passed directly, via environment variable or via command line argument. The latter cases
allow to easily configure multiple environments by having a separate file for each environment.
- :class:`~confz.ConfZEnvSource` allows to load config data from environment variables and .env files. It supports to
- select the corresponding variables with allow- and deny-lists and with an optional prefix. The variable names are
+ select the corresponding variables with allow- and deny-lists and with an optional prefix and optional custom separator for nested variables. The variable names are
either inferred from the config name or can be explicitly mapped.
- :class:`~confz.ConfZCLArgSource` allows to load config data from command line arguments. An optional prefix allows
to select only parts of the arguments. The argument names are either inferred from the config name or can be
diff --git a/tests/loaders/test_env_loader.py b/tests/loaders/test_env_loader.py
index cba4af4..f2f3bda 100644
--- a/tests/loaders/test_env_loader.py
+++ b/tests/loaders/test_env_loader.py
@@ -128,3 +128,15 @@ def test_dotenv_loading(monkeypatch):
assert config.attr2 == 1
assert config.inner.attr1_name == 21
assert config.inner.attr_override == "2002"
+
+
+def test_separator(monkeypatch):
+ monkeypatch.setenv("INNER__ATTR1_NAME", "21")
+ monkeypatch.setenv("INNER__ATTR-OVERRIDE", "2002")
+ monkeypatch.setenv("ATTR2", "1")
+ config = OuterConfig(
+ config_sources=ConfZEnvSource(allow_all=True, nested_separator="__")
+ )
+ assert config.attr2 == 1
+ assert config.inner.attr1_name == 21
+ assert config.inner.attr_override == "2002"
From 314a93e0ab09a051ec26f7248d592b980b09dbbe Mon Sep 17 00:00:00 2001
From: deepaerial <nazarii.oleksiuk@gmail.com>
Date: Mon, 6 Jun 2022 19:55:06 +0200
Subject: [PATCH 2/3] Added optional separator variable for CLI loader class
---
confz/confz_source.py | 2 ++
confz/loaders/cl_arg_loader.py | 4 +++-
confz/loaders/loader.py | 3 ++-
docs/source/usage/sources_loaders.rst | 2 +-
tests/loaders/test_cl_arg_loader.py | 17 +++++++++++++++++
5 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/confz/confz_source.py b/confz/confz_source.py
index e555fc9..5e145fa 100644
--- a/confz/confz_source.py
+++ b/confz/confz_source.py
@@ -88,6 +88,8 @@ class ConfZCLArgSource(ConfZSource):
remap: Optional[Dict[str, str]] = None
"""Certain command line arguments can be mapped to config arguments with a different
name. The map does not need to include the two dashes at the beginning."""
+ nested_separator: str = "."
+ """Separator will be used in nested environment variables."""
@dataclass
diff --git a/confz/loaders/cl_arg_loader.py b/confz/loaders/cl_arg_loader.py
index e32ce81..60a640e 100644
--- a/confz/loaders/cl_arg_loader.py
+++ b/confz/loaders/cl_arg_loader.py
@@ -25,5 +25,7 @@ def populate_config(cls, config: dict, confz_source: ConfZCLArgSource):
cl_args[cl_name] = cl_value
- cl_args = cls.transform_nested_dicts(cl_args)
+ cl_args = cls.transform_nested_dicts(
+ cl_args, separator=confz_source.nested_separator
+ )
cls.update_dict_recursively(config, cl_args)
diff --git a/confz/loaders/loader.py b/confz/loaders/loader.py
index 1c3503b..052f49d 100644
--- a/confz/loaders/loader.py
+++ b/confz/loaders/loader.py
@@ -29,13 +29,14 @@ def update_dict_recursively(cls, original_dict: Dict, update_dict: Dict):
@classmethod
def transform_nested_dicts(
- cls, dict_in: Dict[str, Any], separator: str = "."
+ cls, dict_in: Dict[str, Any], separator: str
) -> Dict[str, Any]:
"""Transform dictionaries into nested dictionaries, using a separator in the
keys as hint.
:param dict_in: A dictionary with string-keys.
:param separator: The string used to separate dict keys.
+ Default value will no longer be set in a future release.
:return: The transformed dictionary, splitting keys at the separator and
creating a new dictionary out of it.
:raises ConfZUpdateException: If dict keys contradict each other.
diff --git a/docs/source/usage/sources_loaders.rst b/docs/source/usage/sources_loaders.rst
index e70202d..e9ce5f1 100644
--- a/docs/source/usage/sources_loaders.rst
+++ b/docs/source/usage/sources_loaders.rst
@@ -18,7 +18,7 @@ There are multiple config sources which support a heterogeneous set of use-cases
select the corresponding variables with allow- and deny-lists and with an optional prefix and optional custom separator for nested variables. The variable names are
either inferred from the config name or can be explicitly mapped.
- :class:`~confz.ConfZCLArgSource` allows to load config data from command line arguments. An optional prefix allows
- to select only parts of the arguments. The argument names are either inferred from the config name or can be
+ to select only parts of the arguments. Optional custom separator for nested variables is also supported. The argument names are either inferred from the config name or can be
explicitly mapped.
- :class:`~confz.ConfZDataSource` allows to define constant config data. This can be very useful for unit tests, see
:ref:`context_manager`.
diff --git a/tests/loaders/test_cl_arg_loader.py b/tests/loaders/test_cl_arg_loader.py
index 9258c43..2368828 100644
--- a/tests/loaders/test_cl_arg_loader.py
+++ b/tests/loaders/test_cl_arg_loader.py
@@ -41,3 +41,20 @@ def test_remap(monkeypatch):
config = OuterConfig(config_sources=ConfZCLArgSource(remap={"val1": "inner.attr1"}))
assert config.inner.attr1 == 1
assert config.attr2 == 2
+
+
+def test_nested_separator(monkeypatch):
+ argv = sys.argv.copy() + [
+ "--conf_inner__attr1",
+ "1",
+ "--conf_attr2",
+ "2",
+ "--attr1",
+ "100",
+ ]
+ monkeypatch.setattr(sys, "argv", argv)
+ config = OuterConfig(
+ config_sources=ConfZCLArgSource(prefix="conf_", nested_separator="__")
+ )
+ assert config.inner.attr1 == 1
+ assert config.attr2 == 2
From a3740402997b2adf7884ab22c4de7d7a8981c7e5 Mon Sep 17 00:00:00 2001
From: deepaerial <nazarii.oleksiuk@gmail.com>
Date: Tue, 7 Jun 2022 10:05:13 +0200
Subject: [PATCH 3/3] Small fixes in docstrings and documentation.
---
confz/confz_source.py | 2 +-
confz/loaders/loader.py | 2 +-
docs/source/usage/sources_loaders.rst | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/confz/confz_source.py b/confz/confz_source.py
index 5e145fa..7da64b7 100644
--- a/confz/confz_source.py
+++ b/confz/confz_source.py
@@ -89,7 +89,7 @@ class ConfZCLArgSource(ConfZSource):
"""Certain command line arguments can be mapped to config arguments with a different
name. The map does not need to include the two dashes at the beginning."""
nested_separator: str = "."
- """Separator will be used in nested environment variables."""
+ """Separator will be used in nested command line arguments."""
@dataclass
diff --git a/confz/loaders/loader.py b/confz/loaders/loader.py
index 052f49d..22c1a25 100644
--- a/confz/loaders/loader.py
+++ b/confz/loaders/loader.py
@@ -29,7 +29,7 @@ def update_dict_recursively(cls, original_dict: Dict, update_dict: Dict):
@classmethod
def transform_nested_dicts(
- cls, dict_in: Dict[str, Any], separator: str
+ cls, dict_in: Dict[str, Any], separator: str = "."
) -> Dict[str, Any]:
"""Transform dictionaries into nested dictionaries, using a separator in the
keys as hint.
diff --git a/docs/source/usage/sources_loaders.rst b/docs/source/usage/sources_loaders.rst
index e9ce5f1..b875f66 100644
--- a/docs/source/usage/sources_loaders.rst
+++ b/docs/source/usage/sources_loaders.rst
@@ -18,7 +18,7 @@ There are multiple config sources which support a heterogeneous set of use-cases
select the corresponding variables with allow- and deny-lists and with an optional prefix and optional custom separator for nested variables. The variable names are
either inferred from the config name or can be explicitly mapped.
- :class:`~confz.ConfZCLArgSource` allows to load config data from command line arguments. An optional prefix allows
- to select only parts of the arguments. Optional custom separator for nested variables is also supported. The argument names are either inferred from the config name or can be
+ to select only parts of the arguments. Optional custom separator for nested command line arguments is also supported. The argument names are either inferred from the config name or can be
explicitly mapped.
- :class:`~confz.ConfZDataSource` allows to define constant config data. This can be very useful for unit tests, see
:ref:`context_manager`.