From fd6f9c008d646cb2136bdf81dc897424df80e0fd Mon Sep 17 00:00:00 2001 From: junhl Date: Sun, 18 Apr 2021 23:57:52 -0400 Subject: [PATCH 1/2] General usage of both LuigiConfigParser and LuigiTomlParser in S3 client --- luigi/contrib/s3.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/luigi/contrib/s3.py b/luigi/contrib/s3.py index 7742053a42..c85cf092a8 100644 --- a/luigi/contrib/s3.py +++ b/luigi/contrib/s3.py @@ -480,9 +480,12 @@ def list(self, path, start_time=None, end_time=None, return_key=False): # backw @staticmethod def _get_s3_config(key=None): - defaults = dict(configuration.get_config().defaults()) + # LuigiConfigParser has defaults since [DEFAULT] is a special section + # LuigiTomlParser does not have it because [DEFAULT] is not a special section + luigi_config = configuration.get_config() + defaults = dict(luigi_config.defaults()) if hasattr(luigi_config, "defaults") else {} try: - config = dict(configuration.get_config().items('s3')) + config = dict(luigi_config['s3']) except (NoSectionError, KeyError): return {} # So what ports etc can be read without us having to specify all dtypes From 869d38d7c260b48c43afafa7f2b8871f87009f20 Mon Sep 17 00:00:00 2001 From: junhl Date: Sun, 18 Apr 2021 23:58:39 -0400 Subject: [PATCH 2/2] Add lacking functions to LuigiTomlParser to match LuigiConfigParser --- luigi/configuration/toml_parser.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/luigi/configuration/toml_parser.py b/luigi/configuration/toml_parser.py index 43a97a08d7..96d19ec6f8 100644 --- a/luigi/configuration/toml_parser.py +++ b/luigi/configuration/toml_parser.py @@ -61,6 +61,23 @@ def read(self, config_paths): return self.data + def defaults(self): + """ + Unlike ConfigParser, [DEFAULT] section is not particularly meaningful in toml. + This is to match LuigiConfigParser. + """ + return {} + + def items(self, section=None): + """ + Return a list of (name, value) tuples for each option in a section. + This is to match LuigiConfigParser. + """ + if section is None: + return self.data.items() + else: + return [(option, value) for option, value in self.data[section].items()] + def get(self, section, option, default=NO_DEFAULT, **kwargs): try: return self.data[section][option]