From 2ea14db6053c095d1a8365da3b2be7a9f9829214 Mon Sep 17 00:00:00 2001 From: Bea Steers Date: Mon, 8 Jun 2020 08:48:15 -0400 Subject: [PATCH 1/4] add lazy config source --- confuse/core.py | 59 +++--------------- confuse/sources.py | 143 +++++++++++++++++++++++++++++++++++++------ test/test_sources.py | 53 ++++++++++++++++ 3 files changed, 187 insertions(+), 68 deletions(-) create mode 100644 test/test_sources.py diff --git a/confuse/core.py b/confuse/core.py index 222ec02..d819b35 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -568,20 +568,18 @@ def _add_user_source(self): """ filename = self.user_config_path() if os.path.isfile(filename): - yaml_data = confuse.load_yaml(filename, loader=self.loader) or {} - self.add(confuse.ConfigSource(yaml_data, filename)) + self.add(confuse.ConfigSource.of(filename, loader=self.loader)) def _add_default_source(self): """Add the package's default configuration settings. This looks for a YAML file located inside the package for the module `modname` if it was given. """ - if self.modname: - if self._package_path: - filename = os.path.join(self._package_path, DEFAULT_FILENAME) - if os.path.isfile(filename): - yaml_data = confuse.load_yaml(filename, loader=self.loader) - self.add(confuse.ConfigSource(yaml_data, filename, True)) + if self._package_path: + filename = os.path.join(self._package_path, DEFAULT_FILENAME) + if os.path.isfile(filename): + self.add(confuse.ConfigSource.of( + filename, loader=self.loader, default=True)) def read(self, user=True, defaults=True): """Find and read the files for this configuration and set them @@ -634,9 +632,8 @@ def set_file(self, filename): """Parses the file as YAML and inserts it into the configuration sources with highest priority. """ - filename = os.path.abspath(filename) - yaml_data = confuse.load_yaml(filename, loader=self.loader) - self.set(confuse.ConfigSource(yaml_data, filename)) + self.set(confuse.ConfigSource.of( + os.path.abspath(filename), loader=self.loader)) def dump(self, full=True, redact=False): """Dump the Configuration object to a YAML file. @@ -684,42 +681,4 @@ class LazyConfig(Configuration): the module level. """ def __init__(self, appname, modname=None): - super(LazyConfig, self).__init__(appname, modname, False) - self._materialized = False # Have we read the files yet? - self._lazy_prefix = [] # Pre-materialization calls to set(). - self._lazy_suffix = [] # Calls to add(). - - def read(self, user=True, defaults=True): - self._materialized = True - super(LazyConfig, self).read(user, defaults) - - def resolve(self): - if not self._materialized: - # Read files and unspool buffers. - self.read() - self.sources += self._lazy_suffix - self.sources[:0] = self._lazy_prefix - return super(LazyConfig, self).resolve() - - def add(self, value): - super(LazyConfig, self).add(value) - if not self._materialized: - # Buffer additions to end. - self._lazy_suffix += self.sources - del self.sources[:] - - def set(self, value): - super(LazyConfig, self).set(value) - if not self._materialized: - # Buffer additions to beginning. - self._lazy_prefix[:0] = self.sources - del self.sources[:] - - def clear(self): - """Remove all sources from this configuration.""" - super(LazyConfig, self).clear() - self._lazy_suffix = [] - self._lazy_prefix = [] - - -# "Validated" configuration views: experimental! + super(LazyConfig, self).__init__(appname, modname, read=False) diff --git a/confuse/sources.py b/confuse/sources.py index b1d5a09..aea4ab1 100644 --- a/confuse/sources.py +++ b/confuse/sources.py @@ -1,16 +1,52 @@ from __future__ import division, absolute_import, print_function +import os +import functools import confuse -__all__ = ['ConfigSource'] +__all__ = ['ConfigSource', 'YamlSource'] + + +UNSET = object() # sentinel + + +def _load_first(func): + '''Call self.load() before the function is called - used for lazy source + loading''' + def inner(self, *a, **kw): + self.load() + return func(self, *a, **kw) + + try: + return functools.wraps(func)(inner) + except AttributeError: + # in v2 they don't ignore missing attributes + # v3: https://github.com/python/cpython/blob/3.8/Lib/functools.py + # v2: https://github.com/python/cpython/blob/2.7/Lib/functools.py + inner.__name__ = func.__name__ + return inner class ConfigSource(dict): - """A dictionary augmented with metadata about the source of the + '''A dictionary augmented with metadata about the source of the configuration. - """ - def __init__(self, value, filename=None, default=False): - super(ConfigSource, self).__init__(value) + ''' + def __getattribute__(self, k): + x = super(ConfigSource, self).__getattribute__(k) + if k == 'keys': + # HACK: in 2.7, it appears that doing dict(source) only checks for + # the existance of a keys attribute and doesn't actually cast + # to a dict, so we never get the chance to load. My goal + # is to remove this entirely ASAP. + x() + return x + + def __init__(self, value=UNSET, filename=None, default=False, + retry=False): + # track whether a config source has been set yet + self.loaded = value is not UNSET + self.retry = retry + super(ConfigSource, self).__init__(value if self.loaded else {}) if (filename is not None and not isinstance(filename, confuse.BASESTRING)): raise TypeError(u'filename must be a string or None') @@ -18,21 +54,92 @@ def __init__(self, value, filename=None, default=False): self.default = default def __repr__(self): - return 'ConfigSource({0!r}, {1!r}, {2!r})'.format( - super(ConfigSource, self), - self.filename, - self.default, - ) + return '{}({}, filename={}, default={})'.format( + self.__class__.__name__, + dict.__repr__(self) + if self.loaded else '[Unloaded]' + if self.exists else "[Source doesn't exist]", + self.filename, self.default) + + @property + def exists(self): + """Does this config have access to usable configuration values?""" + return self.loaded or self.filename and os.path.isfile(self.filename) + + def load(self): + """Ensure that the source is loaded.""" + if not self.loaded: + self.config_dir() + self.loaded = self._load() is not False or not self.retry + return self + + def _load(self): + """Load config from source and update self. + If it doesn't load, return False to keep it marked as unloaded. + Otherwise it will be assumed to be loaded. + """ + + def config_dir(self, create=True): + """Create the config dir, if there's a filename associated with the + source.""" + if self.filename: + dirname = os.path.dirname(self.filename) + if create and dirname and not os.path.isdir(dirname): + os.makedirs(dirname) + return dirname + return None + + # overriding dict methods so that the configuration is loaded before any + # of them are run + __getitem__ = _load_first(dict.__getitem__) + __iter__ = _load_first(dict.__iter__) + # __len__ = _load_first(dict.__len__) + keys = _load_first(dict.keys) + values = _load_first(dict.values) @classmethod - def of(cls, value): - """Given either a dictionary or a `ConfigSource` object, return - a `ConfigSource` object. This lets a function accept either type - of object as an argument. + def isoftype(cls, value, **kw): + return False + + @classmethod + def of(cls, value, **kw): + """Try to convert value to a `ConfigSource` object. This lets a + function accept values that are convertable to a source. """ + # ignore if already a source if isinstance(value, ConfigSource): return value - elif isinstance(value, dict): - return ConfigSource(value) - else: - raise TypeError(u'source value must be a dict') + + # if it's a yaml file + if YamlSource.isoftype(value, **kw): + return YamlSource(value, **kw) + + # if it's an explicit config dict + if isinstance(value, dict): + return ConfigSource(value, **kw) + + # none of the above + raise TypeError( + u'ConfigSource.of value unable to cast to ConfigSource.') + + +class YamlSource(ConfigSource): + """A config source pulled from yaml files.""" + EXTENSIONS = '.yaml', '.yml' + + def __init__(self, filename=None, value=UNSET, optional=False, + loader=confuse.Loader, **kw): + self.optional = optional + self.loader = loader + super(YamlSource, self).__init__(value, filename, **kw) + + @classmethod + def isoftype(cls, value, **kw): + return (isinstance(value, confuse.BASESTRING) + and os.path.splitext(value)[1] in YamlSource.EXTENSIONS) + + def _load(self): + '''Load the file if it exists.''' + if self.optional and not os.path.isfile(self.filename): + return False + self.update(confuse.load_yaml(self.filename, loader=self.loader)) diff --git a/test/test_sources.py b/test/test_sources.py new file mode 100644 index 0000000..bf821d6 --- /dev/null +++ b/test/test_sources.py @@ -0,0 +1,53 @@ +from __future__ import division, absolute_import, print_function + +import confuse +import unittest + + +class ConfigSourceTest(unittest.TestCase): + def _load_yaml(self, file, **kw): + return {'a': 5, 'file': file} + + def setUp(self): + self._orig_load_yaml = confuse.load_yaml + confuse.load_yaml = self._load_yaml + + def tearDown(self): + confuse.load_yaml = self._orig_load_yaml + + def test_source_conversion(self): + # test pure dict source + src = confuse.ConfigSource.of({'a': 5}) + self.assertIsInstance(src, confuse.ConfigSource) + self.assertEqual(src.loaded, True) + # test yaml filename + src = confuse.ConfigSource.of('asdf/asfdd.yml') + self.assertIsInstance(src, confuse.YamlSource) + self.assertEqual(src.loaded, False) + self.assertEqual(src.exists, False) + self.assertEqual(src.config_dir(create=False), 'asdf') + + def test_explicit_load(self): + src = confuse.ConfigSource.of('asdf.yml') + self.assertEqual(src.loaded, False) + src.load() + self.assertEqual(src.loaded, True) + self.assertEqual(src['a'], 5) + + def test_load_getitem(self): + src = confuse.ConfigSource.of('asdf.yml') + self.assertEqual(src.loaded, False) + self.assertEqual(src['a'], 5) + self.assertEqual(src.loaded, True) + + def test_load_cast_dict(self): + src = confuse.ConfigSource.of('asdf.yml') + self.assertEqual(src.loaded, False) + self.assertEqual(dict(src), {'a': 5, 'file': 'asdf.yml'}) + self.assertEqual(src.loaded, True) + + def test_load_keys(self): + src = confuse.ConfigSource.of('asdf.yml') + self.assertEqual(src.loaded, False) + self.assertEqual(set(src.keys()), {'a', 'file'}) + self.assertEqual(src.loaded, True) From e606f140b52ce4034113bb67b71488ae317b5e52 Mon Sep 17 00:00:00 2001 From: Bea Steers Date: Mon, 8 Jun 2020 09:18:04 -0400 Subject: [PATCH 2/4] add and set send kw to cfgsrc.of --- confuse/core.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index d819b35..0454aa2 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -432,11 +432,15 @@ def __init__(self, sources): self.name = ROOT_NAME self.redactions = set() - def add(self, obj): - self.sources.append(confuse.ConfigSource.of(obj)) + def add(self, obj, skip_missing=False, **kw): + src = confuse.ConfigSource.of(obj, **kw) + if skip_missing or src.exists: + self.sources.append(src) - def set(self, value): - self.sources.insert(0, confuse.ConfigSource.of(value)) + def set(self, value, skip_missing=False, **kw): + src = confuse.ConfigSource.of(value, **kw) + if skip_missing or src.exists: + self.sources.insert(0, src) def resolve(self): return ((dict(s), s) for s in self.sources) @@ -566,9 +570,8 @@ def _add_user_source(self): user's configuration directory (given by `config_dir`) if it exists. """ - filename = self.user_config_path() - if os.path.isfile(filename): - self.add(confuse.ConfigSource.of(filename, loader=self.loader)) + self.add( + self.user_config_path(), loader=self.loader, skip_missing=False) def _add_default_source(self): """Add the package's default configuration settings. This looks @@ -576,10 +579,9 @@ def _add_default_source(self): `modname` if it was given. """ if self._package_path: - filename = os.path.join(self._package_path, DEFAULT_FILENAME) - if os.path.isfile(filename): - self.add(confuse.ConfigSource.of( - filename, loader=self.loader, default=True)) + self.add( + os.path.join(self._package_path, DEFAULT_FILENAME), + loader=self.loader, default=True, skip_missing=False) def read(self, user=True, defaults=True): """Find and read the files for this configuration and set them @@ -591,6 +593,8 @@ def read(self, user=True, defaults=True): self._add_user_source() if defaults: self._add_default_source() + for s in self.sources: + s.load() def config_dir(self): """Get the path to the user configuration directory. The @@ -632,8 +636,7 @@ def set_file(self, filename): """Parses the file as YAML and inserts it into the configuration sources with highest priority. """ - self.set(confuse.ConfigSource.of( - os.path.abspath(filename), loader=self.loader)) + self.set(os.path.abspath(filename), loader=self.loader) def dump(self, full=True, redact=False): """Dump the Configuration object to a YAML file. From 407dee3ba25bd4652e864e2fb5d9b15b45607651 Mon Sep 17 00:00:00 2001 From: Bea Steers Date: Mon, 8 Jun 2020 09:20:14 -0400 Subject: [PATCH 3/4] fix condition --- confuse/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confuse/core.py b/confuse/core.py index 0454aa2..9493324 100644 --- a/confuse/core.py +++ b/confuse/core.py @@ -434,12 +434,12 @@ def __init__(self, sources): def add(self, obj, skip_missing=False, **kw): src = confuse.ConfigSource.of(obj, **kw) - if skip_missing or src.exists: + if not skip_missing or src.exists: self.sources.append(src) def set(self, value, skip_missing=False, **kw): src = confuse.ConfigSource.of(value, **kw) - if skip_missing or src.exists: + if not skip_missing or src.exists: self.sources.insert(0, src) def resolve(self): From 80be6383b432651225685af91b36c13d1c6550a3 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 26 Jun 2020 09:44:31 -0400 Subject: [PATCH 4/4] Fix mocking for new package structure --- test/test_sources.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_sources.py b/test/test_sources.py index bf821d6..0aca7a1 100644 --- a/test/test_sources.py +++ b/test/test_sources.py @@ -1,6 +1,7 @@ from __future__ import division, absolute_import, print_function import confuse +import confuse.yaml_util import unittest @@ -9,11 +10,11 @@ def _load_yaml(self, file, **kw): return {'a': 5, 'file': file} def setUp(self): - self._orig_load_yaml = confuse.load_yaml - confuse.load_yaml = self._load_yaml + self._orig_load_yaml = confuse.yaml_util.load_yaml + confuse.yaml_util.load_yaml = self._load_yaml def tearDown(self): - confuse.load_yaml = self._orig_load_yaml + confuse.yaml_util.load_yaml = self._orig_load_yaml def test_source_conversion(self): # test pure dict source