From 788b442d768e03aa6f183a509547649888643eb1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 20 Feb 2011 13:30:39 -0700 Subject: [PATCH 001/769] Add salt python module --- src/saltext/consul/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/saltext/consul/__init__.py diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py new file mode 100644 index 0000000..e69de29 From 48ddf163acb9dc938658eda84031033776753040 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 20 Feb 2011 22:11:43 -0700 Subject: [PATCH 002/769] Set up the initial calling of the minion routines --- src/saltext/consul/__init__.py | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e69de29..95f6a9d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -0,0 +1,47 @@ +''' +Make me some salt! +''' +# Import python libs +import os +import optparse +# Import salt libs +import salt.master +import salt.minion +import salt.utils + +class Master(object): + ''' + Creates a master server + ''' + + +class Minion(object): + ''' + Create a minion server + ''' + def __init__(self): + self.cli = self.__parse_cli() + self.opts = salt.utils.minion_config(self.cli) + + def __parse_cli(self): + ''' + Parse the cli input + ''' + parser = optparse.OptionParser() + parser.add_option('-f', + '--foreground', + dest='foreground', + default=False, + action='store_true', + help='Run the minion in the foreground') + parser.add_option('-c', + '--config', + dest='config', + default='/etc/salt/minion', + help='Pass in an alternative configuration file') + + options, args = parser.parse_args() + cli = {'foreground': options.foreground, + 'config': options.config} + + return cli From ce664672de7cf836d32336ac3beee9c56772be99 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 27 Feb 2011 14:14:26 -0700 Subject: [PATCH 003/769] hook in the minion startup --- src/saltext/consul/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 95f6a9d..7928ce6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -45,3 +45,10 @@ def __parse_cli(self): 'config': options.config} return cli + + def start(self): + ''' + Execute this method to start up a minion. + ''' + minion = salt.Minion(opts) + minion.tune_in() From 75bfec486462626917a350e0e72f592f38c6f4dd Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 27 Feb 2011 15:19:59 -0700 Subject: [PATCH 004/769] Fix incorrect reference to opts dict --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7928ce6..6267491 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -50,5 +50,5 @@ def start(self): ''' Execute this method to start up a minion. ''' - minion = salt.Minion(opts) + minion = salt.Minion(self.opts) minion.tune_in() From 2abf036ab9a608790dab5418d7d5629b2757fe96 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 27 Feb 2011 17:08:34 -0700 Subject: [PATCH 005/769] fix reference to incorrect package in the salt module --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6267491..166a3de 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,6 +7,7 @@ # Import salt libs import salt.master import salt.minion +import salt.config import salt.utils class Master(object): @@ -21,7 +22,7 @@ class Minion(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.utils.minion_config(self.cli) + self.opts = salt.config.minion_config(self.cli) def __parse_cli(self): ''' From 0db0758f4a9d92c8f9b46f4ec197833aaf50d3c3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 10:44:15 -0700 Subject: [PATCH 006/769] add the hooks the initialize a master --- src/saltext/consul/__init__.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 166a3de..cae9568 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -14,6 +14,39 @@ class Master(object): ''' Creates a master server ''' + def __init__(self): + self.cli = self.__parse_cli() + self.opts = salt.config.master_config(self.cli) + + def __parse_cli(self): + ''' + Parse the cli for options passed to a master daemon + ''' + parser = optparse.OptionParser() + parser.add_option('-f', + '--foreground', + dest='foreground', + default=False, + action='store_true', + help='Run the master in the foreground') + parser.add_option('-c', + '--config', + dest='config', + default='/etc/salt/master', + help='Pass in an alternative configuration file') + + options, args = parser.parse_args() + cli = {'foreground': options.foreground, + 'config': options.config} + + return cli + + def start(self): + ''' + Run the sequence to start a salt master server + ''' + pass + class Minion(object): From 2f261b03318722285b4ee3e8598f433710ee6450 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 12:05:42 -0700 Subject: [PATCH 007/769] fix bugs in config file parsing --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cae9568..72424ac 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -16,7 +16,7 @@ class Master(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.config.master_config(self.cli) + self.opts = salt.config.master_config(self.cli['config']) def __parse_cli(self): ''' From e59f51ea507aa2f7e98d1d981bcfdb1b59f77e52 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 12:06:45 -0700 Subject: [PATCH 008/769] fix the config parsing for master and minon --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 72424ac..de3cd6f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -55,7 +55,7 @@ class Minion(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.config.minion_config(self.cli) + self.opts = salt.config.minion_config(self.cli['config']) def __parse_cli(self): ''' From 1620a4aa8f49522b280ba887f769b098866d9636 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 12:09:13 -0700 Subject: [PATCH 009/769] Set hooks to start the master servers --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index de3cd6f..f53da8b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -45,8 +45,8 @@ def start(self): ''' Run the sequence to start a salt master server ''' - pass - + master = salt.Master(self.opts) + master.start() class Minion(object): From 7811d52686450ccb3f83e062fa5a83cd8c127be4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 12:13:24 -0700 Subject: [PATCH 010/769] Reference the rigth salt master class in the chain --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f53da8b..188f39c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -45,7 +45,7 @@ def start(self): ''' Run the sequence to start a salt master server ''' - master = salt.Master(self.opts) + master = salt.master.Master(self.opts) master.start() From 2164fe06c0bf20fdc1809548b932b3bd857fea7a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 12:26:31 -0700 Subject: [PATCH 011/769] Add method to verifyu that the environment pis suitible --- src/saltext/consul/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 188f39c..f6e5f50 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -41,10 +41,19 @@ def __parse_cli(self): return cli + def _verify_env(self): + ''' + Verify that the named direcotries are in place and that the environment + can shake the salt + ''' + if not os.path.isdir(self.opts['cachedir']): + os.makedirs(self.opts['cachedir']) + def start(self): ''' Run the sequence to start a salt master server ''' + self._verify_env() master = salt.master.Master(self.opts) master.start() From 373f7fc5b7d98c61adb29925b8a7d9c9b3b7145e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 15:52:16 -0700 Subject: [PATCH 012/769] write in code so that salt components can run as daemons! --- src/saltext/consul/__init__.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f6e5f50..9f6513d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -23,12 +23,12 @@ def __parse_cli(self): Parse the cli for options passed to a master daemon ''' parser = optparse.OptionParser() - parser.add_option('-f', - '--foreground', - dest='foreground', + parser.add_option('-d', + '--daemon', + dest='daemon', default=False, action='store_true', - help='Run the master in the foreground') + help='Run the master in a daemon') parser.add_option('-c', '--config', dest='config', @@ -36,7 +36,7 @@ def __parse_cli(self): help='Pass in an alternative configuration file') options, args = parser.parse_args() - cli = {'foreground': options.foreground, + cli = {'daemon': options.daemon, 'config': options.config} return cli @@ -55,6 +55,8 @@ def start(self): ''' self._verify_env() master = salt.master.Master(self.opts) + if self.opts['daemon']: + salt.utils.daemonize() master.start() @@ -71,12 +73,12 @@ def __parse_cli(self): Parse the cli input ''' parser = optparse.OptionParser() - parser.add_option('-f', - '--foreground', - dest='foreground', + parser.add_option('-d', + '--daemon', + dest='daemon', default=False, action='store_true', - help='Run the minion in the foreground') + help='Run the minion as a daemon') parser.add_option('-c', '--config', dest='config', @@ -84,7 +86,7 @@ def __parse_cli(self): help='Pass in an alternative configuration file') options, args = parser.parse_args() - cli = {'foreground': options.foreground, + cli = {'daemon': options.daemon, 'config': options.config} return cli @@ -94,4 +96,6 @@ def start(self): Execute this method to start up a minion. ''' minion = salt.Minion(self.opts) + if self.opts['daemon']: + salt.utils.daemonize() minion.tune_in() From d555ce9c33e1e063e924fcd153c3c2ab0e612e84 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 16:00:20 -0700 Subject: [PATCH 013/769] Fix daemon using the wrong option dict --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9f6513d..0bc10f1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -55,7 +55,7 @@ def start(self): ''' self._verify_env() master = salt.master.Master(self.opts) - if self.opts['daemon']: + if self.cli['daemon']: salt.utils.daemonize() master.start() @@ -96,6 +96,6 @@ def start(self): Execute this method to start up a minion. ''' minion = salt.Minion(self.opts) - if self.opts['daemon']: + if self.cli['daemon']: salt.utils.daemonize() minion.tune_in() From d28e9c48fcdc2060152e05e0efe7a62191b3d15c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 16:01:50 -0700 Subject: [PATCH 014/769] fix initial minion object --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0bc10f1..b10c9c9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -95,7 +95,7 @@ def start(self): ''' Execute this method to start up a minion. ''' - minion = salt.Minion(self.opts) + minion = salt.minion.Minion(self.opts) if self.cli['daemon']: salt.utils.daemonize() minion.tune_in() From 5e6f9de83697c681e015fcf458189af7db88d451 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 3 Mar 2011 16:03:29 -0700 Subject: [PATCH 015/769] rename the modules dir to modules --- src/saltext/consul/modules/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/saltext/consul/modules/__init__.py diff --git a/src/saltext/consul/modules/__init__.py b/src/saltext/consul/modules/__init__.py new file mode 100644 index 0000000..e69de29 From a124c29d84945cbb019c57e85b17e1958692878b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 4 Mar 2011 10:22:15 -0700 Subject: [PATCH 016/769] modified the verify_env method to be a function --- src/saltext/consul/__init__.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b10c9c9..26a401d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,6 +10,15 @@ import salt.config import salt.utils +def verify_env(dirs): + ''' + Verify that the named direcotries are in place and that the environment + can shake the salt + ''' + for dir_ in dirs: + if not os.path.isdir(dir_): + os.makedirs(dir_) + class Master(object): ''' Creates a master server @@ -41,19 +50,11 @@ def __parse_cli(self): return cli - def _verify_env(self): - ''' - Verify that the named direcotries are in place and that the environment - can shake the salt - ''' - if not os.path.isdir(self.opts['cachedir']): - os.makedirs(self.opts['cachedir']) - def start(self): ''' Run the sequence to start a salt master server ''' - self._verify_env() + verify_env([self.opts['pki_dir'], self.opts['cachedir']]) master = salt.master.Master(self.opts) if self.cli['daemon']: salt.utils.daemonize() @@ -95,6 +96,7 @@ def start(self): ''' Execute this method to start up a minion. ''' + verify_env([self.opts['pki_dir'], self.opts['cachedir']]) minion = salt.minion.Minion(self.opts) if self.cli['daemon']: salt.utils.daemonize() From ad281462542f2fbdf481b27bffa1913e3c077adf Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 4 Mar 2011 20:48:50 -0700 Subject: [PATCH 017/769] A bunch of small fixes --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 26a401d..8640c7d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -54,7 +54,8 @@ def start(self): ''' Run the sequence to start a salt master server ''' - verify_env([self.opts['pki_dir'], self.opts['cachedir']]) + verify_env([os.path.join(self.opts['pki_dir'], 'minions'), + self.opts['cachedir']]) master = salt.master.Master(self.opts) if self.cli['daemon']: salt.utils.daemonize() From 6a0817abc9072aae00543fba7022a38f7c2cb24d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 23 Mar 2011 22:16:59 -0600 Subject: [PATCH 018/769] Add in pubkey "signing" capabilities --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8640c7d..bbb7733 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -55,6 +55,7 @@ def start(self): Run the sequence to start a salt master server ''' verify_env([os.path.join(self.opts['pki_dir'], 'minions'), + os.path.join(self.opts['pki_dir'], 'minions_pre') self.opts['cachedir']]) master = salt.master.Master(self.opts) if self.cli['daemon']: From 12ab16b26c1bad5cf15c8e33a5f556aa8f932392 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 23 Mar 2011 23:34:50 -0600 Subject: [PATCH 019/769] Daemonize before authenticating, you can always run salt-minion without -d --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index bbb7733..327841d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -99,7 +99,7 @@ def start(self): Execute this method to start up a minion. ''' verify_env([self.opts['pki_dir'], self.opts['cachedir']]) - minion = salt.minion.Minion(self.opts) if self.cli['daemon']: salt.utils.daemonize() + minion = salt.minion.Minion(self.opts) minion.tune_in() From 136223c70dd01dc5a51bb7396fa134c68d846b85 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 23 Mar 2011 23:44:57 -0600 Subject: [PATCH 020/769] fix small syntax issues --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 327841d..cb6f017 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -55,8 +55,8 @@ def start(self): Run the sequence to start a salt master server ''' verify_env([os.path.join(self.opts['pki_dir'], 'minions'), - os.path.join(self.opts['pki_dir'], 'minions_pre') - self.opts['cachedir']]) + os.path.join(self.opts['pki_dir'], 'minions_pre'), + self.opts['cachedir']],) master = salt.master.Master(self.opts) if self.cli['daemon']: salt.utils.daemonize() From 09a9c3044243fa95473194511df4b5b8a8b9fa47 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 26 Mar 2011 13:21:10 -0600 Subject: [PATCH 021/769] Add jobs dir to the autogenerated directories --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cb6f017..d8c9f53 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -56,7 +56,8 @@ def start(self): ''' verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), - self.opts['cachedir']],) + os.path.join(self.opts['cachedir'], 'jobs'), + ) master = salt.master.Master(self.opts) if self.cli['daemon']: salt.utils.daemonize() From 1103b1ec771ea3dd922dc51846b6e78acf4350e6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 26 Mar 2011 15:36:41 -0600 Subject: [PATCH 022/769] Repair some syntax issues --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d8c9f53..7572fc8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -57,7 +57,7 @@ def start(self): verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), - ) + ]) master = salt.master.Master(self.opts) if self.cli['daemon']: salt.utils.daemonize() From 7a49141ae184588a7b93ff9d5b3aa44730ce9d89 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 29 Apr 2011 12:58:00 -0600 Subject: [PATCH 023/769] Add states module directory for modules used to enforce minion states --- src/saltext/consul/states/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/saltext/consul/states/__init__.py diff --git a/src/saltext/consul/states/__init__.py b/src/saltext/consul/states/__init__.py new file mode 100644 index 0000000..e69de29 From 3fa286ac70bc8f9e5a27277ae78b0d213cab5f8c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 26 May 2011 00:00:25 +0100 Subject: [PATCH 024/769] Improve logging support. Changed to a more adequate logging "system". Some other minor changes are still needed, namely the config files. --- src/saltext/consul/__init__.py | 53 ++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7572fc8..a117b1a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -5,10 +5,8 @@ import os import optparse # Import salt libs -import salt.master -import salt.minion import salt.config -import salt.utils + def verify_env(dirs): ''' @@ -31,6 +29,7 @@ def __parse_cli(self): ''' Parse the cli for options passed to a master daemon ''' + import salt.log parser = optparse.OptionParser() parser.add_option('-d', '--daemon', @@ -43,8 +42,19 @@ def __parse_cli(self): dest='config', default='/etc/salt/master', help='Pass in an alternative configuration file') - + parser.add_option('-l', + '--log-level', + dest='log_level', + default='warning', + choices=salt.log.LOG_LEVELS.keys(), + help='Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) + ) + options, args = parser.parse_args() + salt.log.setup_console_logger(options.log_level) + cli = {'daemon': options.daemon, 'config': options.config} @@ -54,12 +64,24 @@ def start(self): ''' Run the sequence to start a salt master server ''' + import salt.log + salt.log.setup_logfile_logger( + self.opts['log_file'], self.opts['log_level'] + ) + import logging + self.opts['logger'] = logging.getLogger('salt.stop-using-me') + print self.opts['log_level'] + print self.opts['logger'].getEffectiveLevel() verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), ]) + # Late import so logging works correctly + import salt.master master = salt.master.Master(self.opts) if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils salt.utils.daemonize() master.start() @@ -76,6 +98,7 @@ def __parse_cli(self): ''' Parse the cli input ''' + import salt.log parser = optparse.OptionParser() parser.add_option('-d', '--daemon', @@ -88,8 +111,17 @@ def __parse_cli(self): dest='config', default='/etc/salt/minion', help='Pass in an alternative configuration file') - + parser.add_option('-l', + '--log-level', + dest='log_level', + default='warning', + choices=salt.log.LOG_LEVELS.keys(), + help='Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) + options, args = parser.parse_args() + salt.log.setup_console_logger(options.log_level) cli = {'daemon': options.daemon, 'config': options.config} @@ -99,8 +131,19 @@ def start(self): ''' Execute this method to start up a minion. ''' + import salt.log + salt.log.setup_logfile_logger( + self.opts['log_file'], self.opts['log_level'] + ) + import logging + self.opts['logger'] = logging.getLogger('salt.stop-using-me') + verify_env([self.opts['pki_dir'], self.opts['cachedir']]) if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils salt.utils.daemonize() + # Late import so logging works correctly + import salt.minion minion = salt.minion.Minion(self.opts) minion.tune_in() From d26f6f4d3f839f603db17be50c38bded9f81a64f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 26 May 2011 11:22:37 +0100 Subject: [PATCH 025/769] Improved Logging Conclusion. Concluded the improved logging stuff; Added `log_granular_levels` which will allow to tweak specific logger's levels. Adapted the default configuration to include the improved logging stuff. --- src/saltext/consul/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a117b1a..cc9a4e5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -68,10 +68,10 @@ def start(self): salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] ) + for name, level in self.opts['log_granular_levels'].iteritems(): + salt.log.set_logger_level(name, level) import logging self.opts['logger'] = logging.getLogger('salt.stop-using-me') - print self.opts['log_level'] - print self.opts['logger'].getEffectiveLevel() verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), @@ -135,6 +135,9 @@ def start(self): salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] ) + for name, level in self.opts['log_granular_levels'].iteritems(): + salt.log.set_logger_level(name, level) + import logging self.opts['logger'] = logging.getLogger('salt.stop-using-me') From 7f5c8562a953a58473f6e507b217a3e5171ba823 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 26 May 2011 12:17:02 +0100 Subject: [PATCH 026/769] Typos, white-space, unused/missing imports, wrapping, PEP-8. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cc9a4e5..8f1753a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,7 +10,7 @@ def verify_env(dirs): ''' - Verify that the named direcotries are in place and that the environment + Verify that the named directories are in place and that the environment can shake the salt ''' for dir_ in dirs: From 9fd47ce35527492afef3c2703aecf3edc76e0928 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 28 May 2011 13:32:05 -0600 Subject: [PATCH 027/769] strip out the last of the old logger --- src/saltext/consul/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8f1753a..daaf094 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -71,7 +71,6 @@ def start(self): for name, level in self.opts['log_granular_levels'].iteritems(): salt.log.set_logger_level(name, level) import logging - self.opts['logger'] = logging.getLogger('salt.stop-using-me') verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), @@ -139,7 +138,6 @@ def start(self): salt.log.set_logger_level(name, level) import logging - self.opts['logger'] = logging.getLogger('salt.stop-using-me') verify_env([self.opts['pki_dir'], self.opts['cachedir']]) if self.cli['daemon']: From 6fd70861cac0783a61c9cbb13a7e8332aab26049 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 2 Jun 2011 21:29:54 -0600 Subject: [PATCH 028/769] Add verification thet the log dir exists --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index daaf094..e598fea 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -74,6 +74,7 @@ def start(self): verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), + os.path.dirname(self.opts['log_file']), ]) # Late import so logging works correctly import salt.master @@ -139,7 +140,9 @@ def start(self): import logging - verify_env([self.opts['pki_dir'], self.opts['cachedir']]) + verify_env([self.opts['pki_dir'], self.opts['cachedir'], + os.path.dirname(self.opts['log_file']), + ]) if self.cli['daemon']: # Late import so logging works correctly import salt.utils From 258cd883fc12784cf09c7f20d6ee9d8d335ca74d Mon Sep 17 00:00:00 2001 From: Travis Cline Date: Mon, 20 Jun 2011 20:25:51 -0500 Subject: [PATCH 029/769] Moved up log_path verification. --- src/saltext/consul/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e598fea..869d056 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -64,6 +64,11 @@ def start(self): ''' Run the sequence to start a salt master server ''' + verify_env([os.path.join(self.opts['pki_dir'], 'minions'), + os.path.join(self.opts['pki_dir'], 'minions_pre'), + os.path.join(self.opts['cachedir'], 'jobs'), + os.path.dirname(self.opts['log_file']), + ]) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] @@ -71,11 +76,6 @@ def start(self): for name, level in self.opts['log_granular_levels'].iteritems(): salt.log.set_logger_level(name, level) import logging - verify_env([os.path.join(self.opts['pki_dir'], 'minions'), - os.path.join(self.opts['pki_dir'], 'minions_pre'), - os.path.join(self.opts['cachedir'], 'jobs'), - os.path.dirname(self.opts['log_file']), - ]) # Late import so logging works correctly import salt.master master = salt.master.Master(self.opts) @@ -131,6 +131,9 @@ def start(self): ''' Execute this method to start up a minion. ''' + verify_env([self.opts['pki_dir'], self.opts['cachedir'], + os.path.dirname(self.opts['log_file']), + ]) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] @@ -140,9 +143,6 @@ def start(self): import logging - verify_env([self.opts['pki_dir'], self.opts['cachedir'], - os.path.dirname(self.opts['log_file']), - ]) if self.cli['daemon']: # Late import so logging works correctly import salt.utils From c9438c39d463ad8a55cd658c0d22bb8aa7a9596e Mon Sep 17 00:00:00 2001 From: Travis Cline Date: Mon, 20 Jun 2011 21:12:22 -0500 Subject: [PATCH 030/769] Added slightly nicer feedback on verify_env failures. --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 869d056..655057b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -15,7 +15,10 @@ def verify_env(dirs): ''' for dir_ in dirs: if not os.path.isdir(dir_): - os.makedirs(dir_) + try: + os.makedirs(dir_) + except OSError, e: + print 'Failed to create directory path "%s" - %s' % (dir_, e) class Master(object): ''' From c65c1abeb6232e1f286ed7720529d67516a11253 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 17 Jul 2011 16:27:25 -0600 Subject: [PATCH 031/769] Add dns checks and move damonization forward in the minion --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 655057b..7ec2cc9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -146,11 +146,11 @@ def start(self): import logging + # Late import so logging works correctly + import salt.minion + minion = salt.minion.Minion(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() - # Late import so logging works correctly - import salt.minion - minion = salt.minion.Minion(self.opts) minion.tune_in() From 749181deed8a0ae08cc7b139bd886199610f2154 Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Fri, 22 Jul 2011 13:09:27 -0600 Subject: [PATCH 032/769] Start the monitor daemon when the minion starts and /etc/salt/minion has 'monitor' config. --- src/saltext/consul/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7ec2cc9..6f189e7 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,8 +2,9 @@ Make me some salt! ''' # Import python libs -import os +import multiprocessing import optparse +import os # Import salt libs import salt.config @@ -149,8 +150,23 @@ def start(self): # Late import so logging works correctly import salt.minion minion = salt.minion.Minion(self.opts) + if self.opts.has_key('monitor'): + multiprocessing.Process( + target=salt.start_monitor, + args=[self.opts, minion.functions]).start() if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() minion.tune_in() + +def start_monitor(opts, functions): + import salt.log + salt.log.setup_logfile_logger(opts['log_file'], opts['log_level']) + for name, level in opts['log_granular_levels'].iteritems(): + salt.log.set_logger_level(name, level) + + # Late import so logging works correctly + import salt.monitor + monitor = salt.monitor.Monitor(opts, functions) + monitor.run() From 101b9f541c747cf698993e12085d274b1493010c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 23 Jul 2011 21:40:50 -0600 Subject: [PATCH 033/769] Add docstring to start_monitor --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6f189e7..d49cbd4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -161,6 +161,10 @@ def start(self): minion.tune_in() def start_monitor(opts, functions): + ''' + Start up a Salt monitor daemon, this function should be run inside of a + multiprocess and prepares the environment for the multiprocess + ''' import salt.log salt.log.setup_logfile_logger(opts['log_file'], opts['log_level']) for name, level in opts['log_granular_levels'].iteritems(): From a00f0586c5f5311da822563b2da560d2dc2d3492 Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Tue, 26 Jul 2011 13:38:36 -0600 Subject: [PATCH 034/769] Use fork to create the monitor process rather than the multiprocessing library since the minion and monitor never interact. --- src/saltext/consul/__init__.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6f189e7..59a42e8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -151,9 +151,7 @@ def start(self): import salt.minion minion = salt.minion.Minion(self.opts) if self.opts.has_key('monitor'): - multiprocessing.Process( - target=salt.start_monitor, - args=[self.opts, minion.functions]).start() + start_monitor(self.opts, minion.functions) if self.cli['daemon']: # Late import so logging works correctly import salt.utils @@ -161,12 +159,13 @@ def start(self): minion.tune_in() def start_monitor(opts, functions): - import salt.log - salt.log.setup_logfile_logger(opts['log_file'], opts['log_level']) - for name, level in opts['log_granular_levels'].iteritems(): - salt.log.set_logger_level(name, level) - - # Late import so logging works correctly - import salt.monitor - monitor = salt.monitor.Monitor(opts, functions) - monitor.run() + pid = os.fork() + try: + if pid > 0: + # child (monitor) process + # Late import so logging works correctly + import salt.monitor + monitor = salt.monitor.Monitor(opts, functions) + monitor.run() + except OSError, ex: + log.error('could not fork new monitor process' ) From 418dadbfffa52a7f20aee87a57a0c515eac3bee4 Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Tue, 26 Jul 2011 13:53:38 -0600 Subject: [PATCH 035/769] remove unnecessary 'import multiprocessing' --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index dc89b69..85e98f1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,7 +2,6 @@ Make me some salt! ''' # Import python libs -import multiprocessing import optparse import os # Import salt libs From 2de12e7c0c37762224e72203491e5221abb4844e Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Wed, 27 Jul 2011 15:57:13 -0600 Subject: [PATCH 036/769] Separate the salt minion from the salt monitor by creating the salt-monitor standalone command. Run 'salt-monitor' rather than 'salt-minion' to start the monitor. --- src/saltext/consul/__init__.py | 78 +++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 85e98f1..89743fb 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -149,28 +149,74 @@ def start(self): # Late import so logging works correctly import salt.minion minion = salt.minion.Minion(self.opts) - if self.opts.has_key('monitor'): - start_monitor(self.opts, minion.functions) if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() minion.tune_in() -def start_monitor(opts, functions): +class Monitor(object): ''' - Start up a Salt monitor daemon. - This function currently forks from the minion because we need - the minion to process command line options, setup global services - like logging, and parse available salt commands. + Create a monitor server ''' - pid = os.fork() - try: - if pid > 0: - # child (monitor) process + def __init__(self): + self.cli = self.__parse_cli() + self.opts = salt.config.minion_config(self.cli['config']) + + def __parse_cli(self): + ''' + Parse the cli input + ''' + import salt.log + parser = optparse.OptionParser() + parser.add_option('-d', + '--daemon', + dest='daemon', + default=False, + action='store_true', + help='Run the monitor as a daemon') + parser.add_option('-c', + '--config', + dest='config', + default='/etc/salt/minion', + help='Pass in an alternative configuration file') + parser.add_option('-l', + '--log-level', + dest='log_level', + default='warning', + choices=salt.log.LOG_LEVELS.keys(), + help='Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) + + options, args = parser.parse_args() + salt.log.setup_console_logger(options.log_level) + cli = {'daemon': options.daemon, + 'config': options.config} + + return cli + + def start(self): + ''' + Execute this method to start up a monitor. + ''' + verify_env([self.opts['pki_dir'], self.opts['cachedir'], + os.path.dirname(self.opts['log_file']), + ]) + import salt.log + salt.log.setup_logfile_logger( + self.opts['log_file'], self.opts['log_level'] + ) + for name, level in self.opts['log_granular_levels'].iteritems(): + salt.log.set_logger_level(name, level) + + import logging + + # Late import so logging works correctly + import salt.monitor + monitor = salt.monitor.Monitor(self.opts) + if self.cli['daemon']: # Late import so logging works correctly - import salt.monitor - monitor = salt.monitor.Monitor(opts, functions) - monitor.run() - except OSError, ex: - log.error('could not fork new monitor process' ) + import salt.utils + salt.utils.daemonize() + monitor.start() From efdada4a754b8ff5f27678d2981ed5e417220bc7 Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Thu, 28 Jul 2011 10:41:28 -0600 Subject: [PATCH 037/769] Move monitor configuration from /etc/salt/minion to /etc/salt/monitor. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 89743fb..f77d1fe 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -161,7 +161,7 @@ class Monitor(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.config.minion_config(self.cli['config']) + self.opts = salt.config.monitor_config(self.cli['config']) def __parse_cli(self): ''' From b6b870f7ba21ed550c02ed316ea218fa12685719 Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Thu, 28 Jul 2011 12:02:29 -0600 Subject: [PATCH 038/769] Change salt-monitor config default to /etc/salt/monitor. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f77d1fe..ba9101e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -178,7 +178,7 @@ def __parse_cli(self): parser.add_option('-c', '--config', dest='config', - default='/etc/salt/minion', + default='/etc/salt/monitor', help='Pass in an alternative configuration file') parser.add_option('-l', '--log-level', From 8f550380fb7057d03f49298f605ec55d11131c71 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 30 Jul 2011 16:35:40 -0600 Subject: [PATCH 039/769] Add hood for the salt syndic --- src/saltext/consul/__init__.py | 87 ++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ba9101e..b5f6bc4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -155,6 +155,93 @@ def start(self): salt.utils.daemonize() minion.tune_in() + +class Syndic(object): + ''' + Create a syndic server + ''' + def __init__(self): + self.cli = self.__parse_cli() + self.opts = self.__parse_opts() + + def __prep_opts(self): + ''' + Generate the opts used by the syndic + ''' + opts = salt.config.master_config(self.cli['master_config']) + opts.update(salt.config.minion_config(self.cli['minion_conifg'])) + if opts.has_key('syndic_master'): + opts['master'] = opts['syndic_master'] + return opts + err = 'The syndic_master needs to be configured in the salt master'\ + + ' config, EXITING!\n' + sys.stderr.write(err) + sys.exit(2) + + def __parse_cli(self): + ''' + Parse the cli for options passed to a master daemon + ''' + import salt.log + parser = optparse.OptionParser() + parser.add_option('-d', + '--daemon', + dest='daemon', + default=False, + action='store_true', + help='Run the master in a daemon') + parser.add_option('--master-config', + dest='master_config', + default='/etc/salt/master', + help='Pass in an alternative master configuration file') + parser.add_option('--minion-config', + dest='minion_config', + default='/etc/salt/minion', + help='Pass in an alternative minion configuration file') + parser.add_option('-l', + '--log-level', + dest='log_level', + default='warning', + choices=salt.log.LOG_LEVELS.keys(), + help='Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) + ) + + options, args = parser.parse_args() + salt.log.setup_console_logger(options.log_level) + + cli = {'daemon': options.daemon, + 'config': options.config} + + return cli + + def start(self): + ''' + Execute this method to start up a syndic. + ''' + verify_env([self.opts['pki_dir'], self.opts['cachedir'], + os.path.dirname(self.opts['log_file']), + ]) + import salt.log + salt.log.setup_logfile_logger( + self.opts['log_file'], self.opts['log_level'] + ) + for name, level in self.opts['log_granular_levels'].iteritems(): + salt.log.set_logger_level(name, level) + + import logging + + # Late import so logging works correctly + import salt.syndic + syndic = salt.syndic.Syndic(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + syndic.tune_in() + + class Monitor(object): ''' Create a monitor server From 43ac2cec162eed0f271d336ef7c789fb3768b332 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 30 Jul 2011 16:44:28 -0600 Subject: [PATCH 040/769] Add granular config file path opts for the syndic --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b5f6bc4..1ffe28a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -169,9 +169,12 @@ def __prep_opts(self): Generate the opts used by the syndic ''' opts = salt.config.master_config(self.cli['master_config']) + opts['_minion_config_path'] = opts['conf_file'] opts.update(salt.config.minion_config(self.cli['minion_conifg'])) if opts.has_key('syndic_master'): opts['master'] = opts['syndic_master'] + opts['_master_config_path'] = opts['conf_file'] + opts.pop('conf_file') return opts err = 'The syndic_master needs to be configured in the salt master'\ + ' config, EXITING!\n' From 76acf113f7297978e9f0ef58ae3c0f857c8f0c40 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 30 Jul 2011 16:47:50 -0600 Subject: [PATCH 041/769] change the conf refs to be more consistent --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 1ffe28a..1ce6cbd 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -169,11 +169,11 @@ def __prep_opts(self): Generate the opts used by the syndic ''' opts = salt.config.master_config(self.cli['master_config']) - opts['_minion_config_path'] = opts['conf_file'] + opts['_minion_conf_file'] = opts['conf_file'] opts.update(salt.config.minion_config(self.cli['minion_conifg'])) if opts.has_key('syndic_master'): opts['master'] = opts['syndic_master'] - opts['_master_config_path'] = opts['conf_file'] + opts['_master_conf_file'] = opts['conf_file'] opts.pop('conf_file') return opts err = 'The syndic_master needs to be configured in the salt master'\ From 76c1cf78b69f6039df3e7562e4e22ed5b6b9f3c9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 30 Jul 2011 22:21:35 -0600 Subject: [PATCH 042/769] remove refs to the salt.syndic module, Syndic is in the minion module --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 1ce6cbd..e9202f6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -236,8 +236,8 @@ def start(self): import logging # Late import so logging works correctly - import salt.syndic - syndic = salt.syndic.Syndic(self.opts) + import salt.minion + syndic = salt.minion.Syndic(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils From b22e8fb19d25c7456d43527839eef625cd9a29ae Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 30 Jul 2011 22:46:45 -0600 Subject: [PATCH 043/769] Repair refs for the syndic --- src/saltext/consul/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e9202f6..54aa41f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -4,6 +4,7 @@ # Import python libs import optparse import os +import sys # Import salt libs import salt.config @@ -162,7 +163,7 @@ class Syndic(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = self.__parse_opts() + self.opts = self.__prep_opts() def __prep_opts(self): ''' @@ -170,7 +171,7 @@ def __prep_opts(self): ''' opts = salt.config.master_config(self.cli['master_config']) opts['_minion_conf_file'] = opts['conf_file'] - opts.update(salt.config.minion_config(self.cli['minion_conifg'])) + opts.update(salt.config.minion_config(self.cli['minion_config'])) if opts.has_key('syndic_master'): opts['master'] = opts['syndic_master'] opts['_master_conf_file'] = opts['conf_file'] @@ -215,7 +216,9 @@ def __parse_cli(self): salt.log.setup_console_logger(options.log_level) cli = {'daemon': options.daemon, - 'config': options.config} + 'minion_config': options.minion_config, + 'master_config': options.master_config, + } return cli From fb80c929b5d927d3765c459824faf1855b8acf8c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 8 Aug 2011 21:36:17 -0600 Subject: [PATCH 044/769] Set more syndic options in the configuration loading --- src/saltext/consul/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 54aa41f..f9fad2d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -173,9 +173,16 @@ def __prep_opts(self): opts['_minion_conf_file'] = opts['conf_file'] opts.update(salt.config.minion_config(self.cli['minion_config'])) if opts.has_key('syndic_master'): + # Some of the opts need to be changed to match the needed opts + # in the minion class. opts['master'] = opts['syndic_master'] + opts['master_ip'] = salt.config.dns_check(opts['master']) + + opts['master_uri'] = 'tcp://' + opts['master_ip'] + ':'\ + + str(opts['master_port']) opts['_master_conf_file'] = opts['conf_file'] opts.pop('conf_file') + print opts return opts err = 'The syndic_master needs to be configured in the salt master'\ + ' config, EXITING!\n' From 8277acb1a34e069d51d8418753473e73cacba2db Mon Sep 17 00:00:00 2001 From: Erik Nolte Date: Wed, 10 Aug 2011 13:54:38 -0600 Subject: [PATCH 045/769] Move remaining salt-monitor code to the salt-monitor project --- src/saltext/consul/__init__.py | 67 ---------------------------------- 1 file changed, 67 deletions(-) mode change 100644 => 100755 src/saltext/consul/__init__.py diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py old mode 100644 new mode 100755 index f9fad2d..3723880 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -253,70 +253,3 @@ def start(self): import salt.utils salt.utils.daemonize() syndic.tune_in() - - -class Monitor(object): - ''' - Create a monitor server - ''' - def __init__(self): - self.cli = self.__parse_cli() - self.opts = salt.config.monitor_config(self.cli['config']) - - def __parse_cli(self): - ''' - Parse the cli input - ''' - import salt.log - parser = optparse.OptionParser() - parser.add_option('-d', - '--daemon', - dest='daemon', - default=False, - action='store_true', - help='Run the monitor as a daemon') - parser.add_option('-c', - '--config', - dest='config', - default='/etc/salt/monitor', - help='Pass in an alternative configuration file') - parser.add_option('-l', - '--log-level', - dest='log_level', - default='warning', - choices=salt.log.LOG_LEVELS.keys(), - help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) - - options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level) - cli = {'daemon': options.daemon, - 'config': options.config} - - return cli - - def start(self): - ''' - Execute this method to start up a monitor. - ''' - verify_env([self.opts['pki_dir'], self.opts['cachedir'], - os.path.dirname(self.opts['log_file']), - ]) - import salt.log - salt.log.setup_logfile_logger( - self.opts['log_file'], self.opts['log_level'] - ) - for name, level in self.opts['log_granular_levels'].iteritems(): - salt.log.set_logger_level(name, level) - - import logging - - # Late import so logging works correctly - import salt.monitor - monitor = salt.monitor.Monitor(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - monitor.start() From fd044681ae97c34e944830928f05a3838ae91ff3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 27 Aug 2011 06:54:52 -0600 Subject: [PATCH 046/769] make the unix socket path configurable --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3723880..40712eb 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -72,6 +72,7 @@ def start(self): os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['cachedir'], 'jobs'), os.path.dirname(self.opts['log_file']), + self.opts['sock_dir'], ]) import salt.log salt.log.setup_logfile_logger( From 6b9c1e96781ff6cacff901e6d6bec086d4609b06 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 27 Aug 2011 10:57:31 -0600 Subject: [PATCH 047/769] don't print the opts when the syndic starts --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 40712eb..194c010 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -183,7 +183,6 @@ def __prep_opts(self): + str(opts['master_port']) opts['_master_conf_file'] = opts['conf_file'] opts.pop('conf_file') - print opts return opts err = 'The syndic_master needs to be configured in the salt master'\ + ' config, EXITING!\n' From e8ec426234316b3a4aa5f14ac0edaa865f87ed73 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 10 Sep 2011 12:20:55 -0600 Subject: [PATCH 048/769] Move minion daemonization before initial authentication so init scripts don't hang --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 194c010..02c6d66 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -150,11 +150,11 @@ def start(self): # Late import so logging works correctly import salt.minion - minion = salt.minion.Minion(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() + minion = salt.minion.Minion(self.opts) minion.tune_in() From 891bd6ea4ab7d65d260cdfa467cedd587d53c657 Mon Sep 17 00:00:00 2001 From: Seth House Date: Sun, 25 Sep 2011 00:30:36 -0600 Subject: [PATCH 049/769] Moved the version string to the Salt module This move will allow fetching the verison from multiple places such as setup.py as well as the docs. --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 02c6d66..10de66c 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,6 +1,9 @@ ''' Make me some salt! ''' +__version_info__ = (0, 9, 2) +__version__ = '.'.join(map(str, __version_info__)) + # Import python libs import optparse import os From 966760476021ebb005b2ab428e116b0a321c7c89 Mon Sep 17 00:00:00 2001 From: Seth House Date: Tue, 4 Oct 2011 18:01:11 -0600 Subject: [PATCH 050/769] Added a --version flag to salt cli interfaces --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 10de66c..e387a2d 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -37,7 +37,7 @@ def __parse_cli(self): Parse the cli for options passed to a master daemon ''' import salt.log - parser = optparse.OptionParser() + parser = optparse.OptionParser(version="%%prog %s" % __version__) parser.add_option('-d', '--daemon', dest='daemon', @@ -107,7 +107,7 @@ def __parse_cli(self): Parse the cli input ''' import salt.log - parser = optparse.OptionParser() + parser = optparse.OptionParser(version="%%prog %s" % __version__) parser.add_option('-d', '--daemon', dest='daemon', @@ -197,7 +197,7 @@ def __parse_cli(self): Parse the cli for options passed to a master daemon ''' import salt.log - parser = optparse.OptionParser() + parser = optparse.OptionParser(version="%%prog %s" % __version__) parser.add_option('-d', '--daemon', dest='daemon', From d5e492544373c649c4286de0bf8fcca0c7087def Mon Sep 17 00:00:00 2001 From: Seth House Date: Sun, 9 Oct 2011 18:38:33 -0600 Subject: [PATCH 051/769] Bump the Salt version number --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 10de66c..a873124 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,7 +1,7 @@ ''' Make me some salt! ''' -__version_info__ = (0, 9, 2) +__version_info__ = (0, 9, 3, 'pre') __version__ = '.'.join(map(str, __version_info__)) # Import python libs From 529a4765c756ab5557e6df10a294082d4e1406ba Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 5 Nov 2011 22:53:13 -0600 Subject: [PATCH 052/769] set the version to 0.9.3 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d1c3f8a..827e1b2 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,7 +1,7 @@ ''' Make me some salt! ''' -__version_info__ = (0, 9, 3, 'pre') +__version_info__ = (0, 9, 3) __version__ = '.'.join(map(str, __version_info__)) # Import python libs From 649d16a415ec34dcbb9658a2dd9e395742e3f20e Mon Sep 17 00:00:00 2001 From: Seth House Date: Sun, 6 Nov 2011 22:46:15 -0700 Subject: [PATCH 053/769] Bump the version string in preparation for the next release --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 827e1b2..5e1ec7f 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,7 +1,7 @@ ''' Make me some salt! ''' -__version_info__ = (0, 9, 3) +__version_info__ = (0, 9, 4, 'pre') __version__ = '.'.join(map(str, __version_info__)) # Import python libs From f9b8969f8d7a0fb431091a28f29417ceafd1b7d9 Mon Sep 17 00:00:00 2001 From: Markus Gattol Date: Sat, 12 Nov 2011 19:32:37 +0000 Subject: [PATCH 054/769] replace has_key() with in Signed-off-by: Markus Gattol --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5e1ec7f..55978dd 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -176,7 +176,7 @@ def __prep_opts(self): opts = salt.config.master_config(self.cli['master_config']) opts['_minion_conf_file'] = opts['conf_file'] opts.update(salt.config.minion_config(self.cli['minion_config'])) - if opts.has_key('syndic_master'): + if 'syndic_master' in opts: # Some of the opts need to be changed to match the needed opts # in the minion class. opts['master'] = opts['syndic_master'] From 0a834781242016b95bfa5dc6ce41e2c3a7c40a1c Mon Sep 17 00:00:00 2001 From: Markus Gattol Date: Sat, 12 Nov 2011 22:31:16 +0000 Subject: [PATCH 055/769] more string formatting Signed-off-by: Markus Gattol --- src/saltext/consul/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 55978dd..729c90a 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,6 +1,7 @@ ''' Make me some salt! ''' + __version_info__ = (0, 9, 4, 'pre') __version__ = '.'.join(map(str, __version_info__)) @@ -8,6 +9,7 @@ import optparse import os import sys + # Import salt libs import salt.config @@ -24,6 +26,7 @@ def verify_env(dirs): except OSError, e: print 'Failed to create directory path "%s" - %s' % (dir_, e) + class Master(object): ''' Creates a master server @@ -182,13 +185,13 @@ def __prep_opts(self): opts['master'] = opts['syndic_master'] opts['master_ip'] = salt.config.dns_check(opts['master']) - opts['master_uri'] = 'tcp://' + opts['master_ip'] + ':'\ - + str(opts['master_port']) + opts['master_uri'] = ('tcp://' + opts['master_ip'] + + ':' + str(opts['master_port'])) opts['_master_conf_file'] = opts['conf_file'] opts.pop('conf_file') return opts - err = 'The syndic_master needs to be configured in the salt master'\ - + ' config, EXITING!\n' + err = ('The syndic_master needs to be configured in the salt master ' + 'config, EXITING!\n') sys.stderr.write(err) sys.exit(2) @@ -217,10 +220,10 @@ def __parse_cli(self): dest='log_level', default='warning', choices=salt.log.LOG_LEVELS.keys(), - help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) - ) + help=('Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) + ) options, args = parser.parse_args() salt.log.setup_console_logger(options.log_level) From 525646fd293446dc80a3dadb9b2254e9bd336123 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 27 Nov 2011 15:32:08 -0700 Subject: [PATCH 056/769] change version to 0.9.4 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 729c90a..69b64d7 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,7 +2,7 @@ Make me some salt! ''' -__version_info__ = (0, 9, 4, 'pre') +__version_info__ = (0, 9, 4) __version__ = '.'.join(map(str, __version_info__)) # Import python libs From 07111725742b5560fde99439011fc089630ad39f Mon Sep 17 00:00:00 2001 From: Seth House Date: Sun, 27 Nov 2011 23:29:45 -0700 Subject: [PATCH 057/769] Bumped version number for the next release --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 69b64d7..3e4d2e1 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,7 +2,7 @@ Make me some salt! ''' -__version_info__ = (0, 9, 4) +__version_info__ = (0, 9, 5, 'pre') __version__ = '.'.join(map(str, __version_info__)) # Import python libs From 6d9fa6c9123fcb97a4f377939d191115b01f86b1 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder Date: Wed, 30 Nov 2011 20:36:25 -0800 Subject: [PATCH 058/769] Hook up the verification module - verify.check_root() to actually get the function - call verify.run() from salt.__init__.verify_env() jeff@desktopmonster:~/src/git/salt/scripts (grains-enhancement)$ python salt-master -l debug 20:40:20,475 [salt.utils.verify][CRITICAL] ZeroMQ python bindings >= 2.1.9 are required jeff@desktopmonster:~/src/git/salt/scripts (grains-enhancement)$ echo $? 1 jeff@desktopmonster:~/src/git/salt/scripts (grains-enhancement)$ python -c 'import zmq; print zmq.__version__' 2.1.7 --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3e4d2e1..a83f3a9 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,6 +12,7 @@ # Import salt libs import salt.config +import salt.utils.verify def verify_env(dirs): @@ -25,6 +26,8 @@ def verify_env(dirs): os.makedirs(dir_) except OSError, e: print 'Failed to create directory path "%s" - %s' % (dir_, e) + # Run the extra verification checks + salt.utils.verify.run() class Master(object): From 863e11e8560b163177186e147491b25e59b38142 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 10 Dec 2011 14:33:57 -0700 Subject: [PATCH 059/769] fix version loading issue with cython --- src/saltext/consul/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a83f3a9..74d0e6d 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,8 +2,6 @@ Make me some salt! ''' -__version_info__ = (0, 9, 5, 'pre') -__version__ = '.'.join(map(str, __version_info__)) # Import python libs import optparse @@ -13,7 +11,7 @@ # Import salt libs import salt.config import salt.utils.verify - +from salt.version import * def verify_env(dirs): ''' From 54062881d0379e90e4aeb9a89269415b69e665c6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 10 Dec 2011 14:45:43 -0700 Subject: [PATCH 060/769] more modifications to fic cython builds --- src/saltext/consul/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 74d0e6d..fafcec3 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,6 +2,8 @@ Make me some salt! ''' +__version_info__ = (0, 9, 5, 'pre') +__version__ = '.'.join(map(str, __version_info__)) # Import python libs import optparse @@ -9,9 +11,11 @@ import sys # Import salt libs -import salt.config -import salt.utils.verify -from salt.version import * +try: + import salt.config + import salt.utils.verify +except ImportError: + pass def verify_env(dirs): ''' From e57577aeac3f32dcd82f2233b713bc9379642e4d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 10 Dec 2011 14:52:35 -0700 Subject: [PATCH 061/769] Add comment explaining try/except block --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fafcec3..c04cd46 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,7 +10,8 @@ import os import sys -# Import salt libs +# Import salt libs, the try block bypasses an issue at build time so that c +# modules don't cause the build to fail try: import salt.config import salt.utils.verify From 0ff5c8cfa9b768243ef2f2467792711c6ee884e1 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder Date: Tue, 27 Dec 2011 17:00:52 -0800 Subject: [PATCH 062/769] Handle Ctrl-c on the syndic or regular minion and log it - Log a message when the syndic or minion is exiting - Cleanly exit and use the same message on all daemons - Add places to handle a sigterm or sighup by KeyboardInterrupt --- src/saltext/consul/__init__.py | 35 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c04cd46..a0b9637 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -93,6 +93,7 @@ def start(self): for name, level in self.opts['log_granular_levels'].iteritems(): salt.log.set_logger_level(name, level) import logging + log = logging.getLogger(__name__) # Late import so logging works correctly import salt.master master = salt.master.Master(self.opts) @@ -162,12 +163,17 @@ def start(self): # Late import so logging works correctly import salt.minion - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - minion = salt.minion.Minion(self.opts) - minion.tune_in() + log = logging.getLogger(__name__) + try: + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + minion = salt.minion.Minion(self.opts) + minion.tune_in() + except KeyboardInterrupt: + log.warn('Stopping the Salt Minion') + raise SystemExit('\nExiting on Ctrl-c') class Syndic(object): @@ -259,9 +265,14 @@ def start(self): # Late import so logging works correctly import salt.minion - syndic = salt.minion.Syndic(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - syndic.tune_in() + log = logging.getLogger(__name__) + try: + syndic = salt.minion.Syndic(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + syndic.tune_in() + except KeyboardInterrupt: + log.warn('Stopping the Salt Syndic Minion') + raise SystemExit('\nExiting on Ctrl-c') From e44824976f44d30222ae17ba0ed21bc1c881650a Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Tue, 3 Jan 2012 09:48:14 +0100 Subject: [PATCH 063/769] Fixed issue #411: Don't bypass real import errors during install, only the _msgpack one --- src/saltext/consul/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a0b9637..e7529b8 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -15,8 +15,9 @@ try: import salt.config import salt.utils.verify -except ImportError: - pass +except ImportError as e: + if e.message != 'No module named _msgpack': + raise def verify_env(dirs): ''' From a455ec8cfce82c778d9961943ca3230e8291e1dd Mon Sep 17 00:00:00 2001 From: Seth House Date: Thu, 22 Dec 2011 15:56:33 -0700 Subject: [PATCH 064/769] Moved version info to own file to avoid import issues Fixes issue #387. --- src/saltext/consul/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e7529b8..9303dce 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,9 +1,7 @@ ''' Make me some salt! ''' - -__version_info__ = (0, 9, 5, 'pre') -__version__ = '.'.join(map(str, __version_info__)) +from salt.version import __version__ # Import python libs import optparse From effe95d2b5014bcf3b4669401c62212dddc25046 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 4 Jan 2012 20:20:38 -0700 Subject: [PATCH 065/769] Add extension modules data to the config system --- src/saltext/consul/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e7529b8..eb86d3f 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -150,8 +150,10 @@ def start(self): ''' Execute this method to start up a minion. ''' - verify_env([self.opts['pki_dir'], self.opts['cachedir'], - os.path.dirname(self.opts['log_file']), + verify_env([self.opts['pki_dir'], + self.opts['cachedir'], + self.opts['extension_modules'], + os.path.dirname(self.opts['log_file']), ]) import salt.log salt.log.setup_logfile_logger( From 690bfd38e0d4bbedb6c295f20b710f4900e0363c Mon Sep 17 00:00:00 2001 From: Jeff Schroeder Date: Sat, 7 Jan 2012 22:02:04 -0800 Subject: [PATCH 066/769] Make the m{aster,inion} log to stdout as they did previously - Introduces the ability to override the log format and date format strings to setup_console_logger(). This might be a good candidate for a fancy configuration option later on. --- src/saltext/consul/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b23bc72..3fbe829 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -66,9 +66,9 @@ def __parse_cli(self): 'see the config file. Default: \'%%default\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) ) - + log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level) + salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, 'config': options.config} @@ -138,7 +138,8 @@ def __parse_cli(self): ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level) + log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' + salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, 'config': options.config} From b41def2bb7dd590b74382e35f8a3c18d750ed893 Mon Sep 17 00:00:00 2001 From: Brad Barden Date: Tue, 10 Jan 2012 20:16:30 -0600 Subject: [PATCH 067/769] protect all verify_env dirs --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3fbe829..31a98d0 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -25,9 +25,12 @@ def verify_env(dirs): for dir_ in dirs: if not os.path.isdir(dir_): try: + cumask = os.umask(191) os.makedirs(dir_) + os.umask(cumask) except OSError, e: print 'Failed to create directory path "%s" - %s' % (dir_, e) + os.chmod(dir_, 448) # Run the extra verification checks salt.utils.verify.run() From 95b70616b414d934de3fceeeced5ac5ecbd844f5 Mon Sep 17 00:00:00 2001 From: Jeff Schroeder Date: Thu, 12 Jan 2012 18:51:39 -0800 Subject: [PATCH 068/769] Don't traceback when failing to chmod directories --- src/saltext/consul/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 31a98d0..909b430 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -4,9 +4,10 @@ from salt.version import __version__ # Import python libs -import optparse import os import sys +import stat +import optparse # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail @@ -29,8 +30,14 @@ def verify_env(dirs): os.makedirs(dir_) os.umask(cumask) except OSError, e: - print 'Failed to create directory path "%s" - %s' % (dir_, e) - os.chmod(dir_, 448) + sys.stderr.write('Failed to create directory path "{0}" - {1}\n'.format(dir_, e)) + + mode = os.stat(dir_) + # TODO: Should this log if it can't set the permissions + # to very secure for these PKI cert directories? + if not stat.S_IMODE(mode.st_mode) == 448: + if os.access(dir_, os.W_OK): + os.chmod(dir_, 448) # Run the extra verification checks salt.utils.verify.run() From aa685a0cb9257b0b594a41455d209df03cc31ecb Mon Sep 17 00:00:00 2001 From: Jeff Bauer Date: Tue, 17 Jan 2012 09:20:17 -0600 Subject: [PATCH 069/769] umask should permit user write permission (077) --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 909b430..4b8dcaf 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -26,7 +26,7 @@ def verify_env(dirs): for dir_ in dirs: if not os.path.isdir(dir_): try: - cumask = os.umask(191) + cumask = os.umask(63) # 077 os.makedirs(dir_) os.umask(cumask) except OSError, e: From 9b762b81954560df5618461522709e36e88779ed Mon Sep 17 00:00:00 2001 From: Corey Quinn Date: Tue, 17 Jan 2012 11:00:04 -0800 Subject: [PATCH 070/769] Pathing --- src/saltext/consul/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/saltext/consul/__init__.py diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py old mode 100755 new mode 100644 From 33e93381f6de3a9c002f1b87da60205b155c0be2 Mon Sep 17 00:00:00 2001 From: Corey Quinn Date: Mon, 16 Jan 2012 15:36:59 -0800 Subject: [PATCH 071/769] Debian/Ubuntu package for upstream acceptance Applying latest changes for packaging Fixed typos in man pages Latest round of changes More packaging fixups Fixed salt-common typo Fixed wildcarding in install files Removed extra man pages Removed trailing slash Fixed links Moved binaries to proper packages Fixed man pages Pathing Perms tweak Missing files Fixed spacing Fixed another lintian error build the msgpack stuff Updating rules and install files Fixed shebang Control updates Fixed copyright file Fixed lintian Fixed overrides cmd.retcode no longer uses subprocess.call since it is broken fix issue with source_hash and trailing whitespace Bye-bye pickle, hello msgpack Add docs for new source powers Add support for source_hash to be a source hash string add pure python template type add return clarifying that no states were found if no states are found change some strings to use format add code to cache jobs on the minion if option is set serialize cache data was caching the wrong line data Add cache_jobs to the minion config template add docs for new config param cache_jobs make the minions return to the master in addition to returning to returners Add capability to designate multiple returns only run the apache module if apachectl is installed only load solr module if solr is installed Debug statement used the wrong variable. Only load nginx on machines that have nginx installed Make it more like the apache module --- src/saltext/consul/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/saltext/consul/__init__.py diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py old mode 100755 new mode 100644 From f5d028b98d6a8c237764b7dc92c64f42b3898040 Mon Sep 17 00:00:00 2001 From: Jeff Bauer Date: Wed, 18 Jan 2012 14:29:38 -0600 Subject: [PATCH 072/769] run salt in user space --- src/saltext/consul/__init__.py | 112 ++++++++++++++++++++++++--------- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4b8dcaf..a73c8ee 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -8,6 +8,7 @@ import sys import stat import optparse +import getpass # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail @@ -42,6 +43,34 @@ def verify_env(dirs): salt.utils.verify.run() +def check_user(user, log): + ''' + Check user and assign process uid/gid. + ''' + if 'os' in os.environ: + if os.environ['os'].startswith('Windows'): + return True + if user == getpass.getuser(): + return True + import pwd # after confirming not running Windows + try: + p = pwd.getpwnam(user) + try: + os.setgid(p.pw_gid) + os.setuid(p.pw_uid) + except OSError: + if user == 'root': + msg = 'Sorry, the salt must run as root. http://xkcd.com/838' + else: + msg = 'Salt must be run from root or user "{0}"'.format(user) + log.critical(msg) + return False + except KeyError: + msg = 'User not found: "{0}"'.format(user) + log.critical(msg) + return False + return True + class Master(object): ''' Creates a master server @@ -49,6 +78,9 @@ class Master(object): def __init__(self): self.cli = self.__parse_cli() self.opts = salt.config.master_config(self.cli['config']) + # command line overrides config + if self.cli['user']: + self.opts['user'] = self.cli['user'] def __parse_cli(self): ''' @@ -67,6 +99,10 @@ def __parse_cli(self): dest='config', default='/etc/salt/master', help='Pass in an alternative configuration file') + parser.add_option('-u', + '--user', + dest='user', + help='Specify user to run minion') parser.add_option('-l', '--log-level', dest='log_level', @@ -104,13 +140,14 @@ def start(self): import logging log = logging.getLogger(__name__) # Late import so logging works correctly - import salt.master - master = salt.master.Master(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - master.start() + if check_user(self.opts['user'], log): + import salt.master + master = salt.master.Master(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + master.start() class Minion(object): @@ -120,6 +157,9 @@ class Minion(object): def __init__(self): self.cli = self.__parse_cli() self.opts = salt.config.minion_config(self.cli['config']) + # command line overrides config + if self.cli['user']: + self.opts['user'] = self.cli['user'] def __parse_cli(self): ''' @@ -138,6 +178,10 @@ def __parse_cli(self): dest='config', default='/etc/salt/minion', help='Pass in an alternative configuration file') + parser.add_option('-u', + '--user', + dest='user', + help='Specify user to run minion') parser.add_option('-l', '--log-level', dest='log_level', @@ -151,7 +195,8 @@ def __parse_cli(self): log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, - 'config': options.config} + 'config': options.config, + 'user': options.user} return cli @@ -170,22 +215,21 @@ def start(self): ) for name, level in self.opts['log_granular_levels'].iteritems(): salt.log.set_logger_level(name, level) - import logging - # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) - try: - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - minion = salt.minion.Minion(self.opts) - minion.tune_in() - except KeyboardInterrupt: - log.warn('Stopping the Salt Minion') - raise SystemExit('\nExiting on Ctrl-c') + if check_user(self.opts['user'], log): + try: + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + minion = salt.minion.Minion(self.opts) + minion.tune_in() + except KeyboardInterrupt: + log.warn('Stopping the Salt Minion') + raise SystemExit('\nExiting on Ctrl-c') class Syndic(object): @@ -195,6 +239,9 @@ class Syndic(object): def __init__(self): self.cli = self.__parse_cli() self.opts = self.__prep_opts() + # command line overrides config + if self.cli['user']: + self.opts['user'] = self.cli['user'] def __prep_opts(self): ''' @@ -239,6 +286,10 @@ def __parse_cli(self): dest='minion_config', default='/etc/salt/minion', help='Pass in an alternative minion configuration file') + parser.add_option('-u', + '--user', + dest='user', + help='Specify user to run minion') parser.add_option('-l', '--log-level', dest='log_level', @@ -278,13 +329,14 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) - try: - syndic = salt.minion.Syndic(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - syndic.tune_in() - except KeyboardInterrupt: - log.warn('Stopping the Salt Syndic Minion') - raise SystemExit('\nExiting on Ctrl-c') + if check_user(self.opts['user'], log): + try: + syndic = salt.minion.Syndic(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + syndic.tune_in() + except KeyboardInterrupt: + log.warn('Stopping the Salt Syndic Minion') + raise SystemExit('\nExiting on Ctrl-c') From 03e46b3830106ab9979f62744b678dde95c84a62 Mon Sep 17 00:00:00 2001 From: Jeff Bauer Date: Wed, 18 Jan 2012 15:07:57 -0600 Subject: [PATCH 073/769] add user to master config --- src/saltext/consul/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a73c8ee..b30cfdb 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -117,7 +117,8 @@ def __parse_cli(self): salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, - 'config': options.config} + 'config': options.config, + 'user': options.user} return cli @@ -306,7 +307,7 @@ def __parse_cli(self): cli = {'daemon': options.daemon, 'minion_config': options.minion_config, 'master_config': options.master_config, - } + 'user': options.user} return cli From e7fd382c566766b8f9088d8bd999d8da00962fcf Mon Sep 17 00:00:00 2001 From: Jeff Bauer Date: Thu, 19 Jan 2012 11:31:04 -0600 Subject: [PATCH 074/769] salt-key option to reject keys --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b30cfdb..d58be2d 100755 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -128,6 +128,7 @@ def start(self): ''' verify_env([os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), + os.path.join(self.opts['pki_dir'], 'minions_rejected'), os.path.join(self.opts['cachedir'], 'jobs'), os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], From 6952f68f76a43474168cd8705b2ff9be71c7aca9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 25 Jan 2012 10:01:44 -0700 Subject: [PATCH 075/769] Add pidfile support --- src/saltext/consul/__init__.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d58be2d..32d12ec 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -19,6 +19,21 @@ if e.message != 'No module named _msgpack': raise + +def set_pidfile(pidfile): + ''' + Save the pidfile + ''' + pdir = os.path.dirname(pidfile) + if not os.path.isdir(pdir): + os.makedirs(pdir) + try: + open(pidfile, 'w+').write(str(os.getpid())) + except IOError: + err = ('Failed to commit the pid file to location {0}, please verify' + ' that the location is available').format(pidfile) + + def verify_env(dirs): ''' Verify that the named directories are in place and that the environment @@ -103,6 +118,11 @@ def __parse_cli(self): '--user', dest='user', help='Specify user to run minion') + parser.add_option('--pid-file', + dest='pidfile', + default='/var/run/salt-master.pid', + help=('Specify the location of the pidfile, default is' + ' /var/run/salt-master.pid')) parser.add_option('-l', '--log-level', dest='log_level', @@ -118,7 +138,8 @@ def __parse_cli(self): cli = {'daemon': options.daemon, 'config': options.config, - 'user': options.user} + 'user': options.user, + 'pidfile': options.pidfile} return cli @@ -133,6 +154,7 @@ def start(self): os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], ]) + set_pidfile(cli['pidfile']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] From 77a108cb219404ab1782469f643414185135d7f5 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 25 Jan 2012 10:03:13 -0700 Subject: [PATCH 076/769] fix cli ref --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 32d12ec..b77d346 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -154,7 +154,7 @@ def start(self): os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], ]) - set_pidfile(cli['pidfile']) + set_pidfile(self.cli['pidfile']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] From 6893fc657cacdaaf5fd5eebb5da646f5fbf6db21 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 25 Jan 2012 10:23:05 -0700 Subject: [PATCH 077/769] Some polishing to pidfile stuff --- src/saltext/consul/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b77d346..553fc5e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -32,6 +32,7 @@ def set_pidfile(pidfile): except IOError: err = ('Failed to commit the pid file to location {0}, please verify' ' that the location is available').format(pidfile) + log.error(err) def verify_env(dirs): @@ -121,8 +122,8 @@ def __parse_cli(self): parser.add_option('--pid-file', dest='pidfile', default='/var/run/salt-master.pid', - help=('Specify the location of the pidfile, default is' - ' /var/run/salt-master.pid')) + help=('Specify the location of the pidfile. Default' + ' %default')) parser.add_option('-l', '--log-level', dest='log_level', @@ -154,7 +155,6 @@ def start(self): os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], ]) - set_pidfile(self.cli['pidfile']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] @@ -166,6 +166,7 @@ def start(self): # Late import so logging works correctly if check_user(self.opts['user'], log): import salt.master + set_pidfile(self.cli['pidfile']) master = salt.master.Master(self.opts) if self.cli['daemon']: # Late import so logging works correctly From b5a3b49bbe0a0fe12d040ad00250c0b252271903 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 26 Jan 2012 15:33:51 -0700 Subject: [PATCH 078/769] Add pidfile support to minion and syndic --- src/saltext/consul/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 553fc5e..05122a3 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -207,6 +207,11 @@ def __parse_cli(self): '--user', dest='user', help='Specify user to run minion') + parser.add_option('--pid-file', + dest='pidfile', + default='/var/run/salt-minion.pid', + help=('Specify the location of the pidfile. Default' + ' %default')) parser.add_option('-l', '--log-level', dest='log_level', @@ -221,7 +226,8 @@ def __parse_cli(self): salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, 'config': options.config, - 'user': options.user} + 'user': options.user, + 'pidfile': options.pidfile} return cli @@ -243,6 +249,7 @@ def start(self): import logging # Late import so logging works correctly import salt.minion + set_pidfile(self.cli['pidfile']) log = logging.getLogger(__name__) if check_user(self.opts['user'], log): try: @@ -315,6 +322,11 @@ def __parse_cli(self): '--user', dest='user', help='Specify user to run minion') + parser.add_option('--pid-file', + dest='pidfile', + default='/var/run/salt-syndic.pid', + help=('Specify the location of the pidfile. Default' + ' %default')) parser.add_option('-l', '--log-level', dest='log_level', @@ -353,6 +365,7 @@ def start(self): # Late import so logging works correctly import salt.minion + set_pidfile(self.cli['pidfile']) log = logging.getLogger(__name__) if check_user(self.opts['user'], log): try: From 2d8370bf58d09c70c5940de28c2e348058e4a217 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 26 Jan 2012 18:00:07 -0700 Subject: [PATCH 079/769] fix pidfile creation time --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 05122a3..0f78e64 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -166,12 +166,12 @@ def start(self): # Late import so logging works correctly if check_user(self.opts['user'], log): import salt.master - set_pidfile(self.cli['pidfile']) master = salt.master.Master(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) master.start() @@ -249,7 +249,6 @@ def start(self): import logging # Late import so logging works correctly import salt.minion - set_pidfile(self.cli['pidfile']) log = logging.getLogger(__name__) if check_user(self.opts['user'], log): try: @@ -257,6 +256,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) minion = salt.minion.Minion(self.opts) minion.tune_in() except KeyboardInterrupt: @@ -365,7 +365,6 @@ def start(self): # Late import so logging works correctly import salt.minion - set_pidfile(self.cli['pidfile']) log = logging.getLogger(__name__) if check_user(self.opts['user'], log): try: @@ -374,6 +373,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) syndic.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Syndic Minion') From 5669781770116992f76c20a5a30cbc1ca0d35850 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 Feb 2012 17:46:25 -0700 Subject: [PATCH 080/769] Add all master processes to pidfile --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0f78e64..b998fef 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -97,6 +97,8 @@ def __init__(self): # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] + # Send the pidfile location to the opts + self.opts['pidfile'] = self.cli['pidfile'] def __parse_cli(self): ''' From 30ef7e2d3ce6dcef450a38c7f3180dd52366913a Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Fri, 10 Feb 2012 15:18:11 +0000 Subject: [PATCH 081/769] Fix copy-paste grammar errors in daemons help --- src/saltext/consul/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b998fef..dba74aa 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -111,7 +111,7 @@ def __parse_cli(self): dest='daemon', default=False, action='store_true', - help='Run the master in a daemon') + help='Run the master as a daemon') parser.add_option('-c', '--config', dest='config', @@ -120,7 +120,7 @@ def __parse_cli(self): parser.add_option('-u', '--user', dest='user', - help='Specify user to run minion') + help='Specify user to run master') parser.add_option('--pid-file', dest='pidfile', default='/var/run/salt-master.pid', @@ -302,7 +302,7 @@ def __prep_opts(self): def __parse_cli(self): ''' - Parse the cli for options passed to a master daemon + Parse the cli for options passed to a syndic daemon ''' import salt.log parser = optparse.OptionParser(version="%%prog %s" % __version__) @@ -311,7 +311,7 @@ def __parse_cli(self): dest='daemon', default=False, action='store_true', - help='Run the master in a daemon') + help='Run the syndic as a daemon') parser.add_option('--master-config', dest='master_config', default='/etc/salt/master', @@ -323,7 +323,7 @@ def __parse_cli(self): parser.add_option('-u', '--user', dest='user', - help='Specify user to run minion') + help='Specify user to run syndic') parser.add_option('--pid-file', dest='pidfile', default='/var/run/salt-syndic.pid', From b413fab6c2f6a21c1acc08de5de961c2ab060b2b Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 15 Feb 2012 10:51:15 -0700 Subject: [PATCH 082/769] remove faulty log entry, we don't have a logger in here --- src/saltext/consul/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index dba74aa..4b0f526 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -30,9 +30,7 @@ def set_pidfile(pidfile): try: open(pidfile, 'w+').write(str(os.getpid())) except IOError: - err = ('Failed to commit the pid file to location {0}, please verify' - ' that the location is available').format(pidfile) - log.error(err) + pass def verify_env(dirs): From d8cd77056dd406fb692c3043a17716b92704b0f8 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 27 Feb 2012 14:01:40 -0700 Subject: [PATCH 083/769] pidfile missing from cli struct --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4b0f526..cacee98 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -343,6 +343,7 @@ def __parse_cli(self): cli = {'daemon': options.daemon, 'minion_config': options.minion_config, 'master_config': options.master_config, + 'pidfile': options.pidfile, 'user': options.user} return cli From 02ceccedd7a9563d53cc7fcf76e27a9121ffac56 Mon Sep 17 00:00:00 2001 From: Gordon McAllister Date: Tue, 28 Feb 2012 16:01:48 -0800 Subject: [PATCH 084/769] Fix a few typos and docstring/comment formatting --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cacee98..5151802 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -49,7 +49,7 @@ def verify_env(dirs): mode = os.stat(dir_) # TODO: Should this log if it can't set the permissions - # to very secure for these PKI cert directories? + # to very secure for these PKI cert directories? if not stat.S_IMODE(mode.st_mode) == 448: if os.access(dir_, os.W_OK): os.chmod(dir_, 448) From d4b711c123e4575b0313df5769f766d521cee6e3 Mon Sep 17 00:00:00 2001 From: Grier Johnson Date: Thu, 1 Mar 2012 11:23:05 -0800 Subject: [PATCH 085/769] Updating multiprocessing child clean-up Fixing minor issues in the first pass for the multiprocessing child clean-up code. --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5151802..0680153 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -172,7 +172,10 @@ def start(self): import salt.utils salt.utils.daemonize() set_pidfile(self.cli['pidfile']) - master.start() + try: + master.start() + except salt.master.MasterExit: + sys.exit() class Minion(object): From 0937a40faf19c8483f2fce5b2baae40c4c5cd0b0 Mon Sep 17 00:00:00 2001 From: Dan Colish Date: Fri, 2 Mar 2012 21:33:20 -0800 Subject: [PATCH 086/769] Allow pidfile to be configured for master in master config in addition to the command line --- src/saltext/consul/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5151802..384b66d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -96,7 +96,8 @@ def __init__(self): if self.cli['user']: self.opts['user'] = self.cli['user'] # Send the pidfile location to the opts - self.opts['pidfile'] = self.cli['pidfile'] + if self.cli['pidfile']: + self.opts['pidfile'] = self.cli['pidfile'] def __parse_cli(self): ''' @@ -121,9 +122,7 @@ def __parse_cli(self): help='Specify user to run master') parser.add_option('--pid-file', dest='pidfile', - default='/var/run/salt-master.pid', - help=('Specify the location of the pidfile. Default' - ' %default')) + help=('Specify the location of the pidfile.')) parser.add_option('-l', '--log-level', dest='log_level', @@ -171,7 +170,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) + set_pidfile(self.opts['pidfile']) master.start() From ef4b289213b5822e654d2ac5b83b00d32b9943eb Mon Sep 17 00:00:00 2001 From: Dan Colish Date: Fri, 2 Mar 2012 23:12:28 -0800 Subject: [PATCH 087/769] Use context manager so file is guaranteed to close after block --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 384b66d..ff57fc5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -28,7 +28,8 @@ def set_pidfile(pidfile): if not os.path.isdir(pdir): os.makedirs(pdir) try: - open(pidfile, 'w+').write(str(os.getpid())) + with open(pidfile, 'w+') as f: + f.write(str(os.getpid())) except IOError: pass From 673dcfda65d022fe5c6b746ad0d67673b197e8d3 Mon Sep 17 00:00:00 2001 From: Grier Johnson Date: Tue, 6 Mar 2012 18:26:23 -0800 Subject: [PATCH 088/769] Adding Range Support to Salt Range is a cluster based metadata store. It was used internally at yahoo and recently open source. You can read about range here: https://github.com/grierj/range/wiki/Introduction-to-Range-with-YAML-files Range allows arbitrary grouping of hosts via clusters and the storage of metadata along with them. Additionally there is a powerful DSL that allows for set operations, regexes, and multilayer cluster referencing. The implementation here runs the range query and reduces it to a list style expression format, which means only the salt master needs to know about range or have the range libraries installed. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 1e62738..9e23e49 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -171,7 +171,7 @@ def start(self): # Late import so logging works correctly import salt.utils salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) + set_pidfile(self.opts['pidfile']) try: master.start() except salt.master.MasterExit: From c920589f7ce58591d52552d021f9eb76d8ff2d53 Mon Sep 17 00:00:00 2001 From: Dan Colish Date: Thu, 8 Mar 2012 20:26:11 -0800 Subject: [PATCH 089/769] Move utility functions into utils --- src/saltext/consul/__init__.py | 71 +--------------------------------- 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9e23e49..fd05304 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,86 +6,19 @@ # Import python libs import os import sys -import stat import optparse -import getpass # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail try: import salt.config - import salt.utils.verify + from salt.utils.process import set_pidfile + from salt.utils.verify import check_user, verify_env except ImportError as e: if e.message != 'No module named _msgpack': raise -def set_pidfile(pidfile): - ''' - Save the pidfile - ''' - pdir = os.path.dirname(pidfile) - if not os.path.isdir(pdir): - os.makedirs(pdir) - try: - with open(pidfile, 'w+') as f: - f.write(str(os.getpid())) - except IOError: - pass - - -def verify_env(dirs): - ''' - Verify that the named directories are in place and that the environment - can shake the salt - ''' - for dir_ in dirs: - if not os.path.isdir(dir_): - try: - cumask = os.umask(63) # 077 - os.makedirs(dir_) - os.umask(cumask) - except OSError, e: - sys.stderr.write('Failed to create directory path "{0}" - {1}\n'.format(dir_, e)) - - mode = os.stat(dir_) - # TODO: Should this log if it can't set the permissions - # to very secure for these PKI cert directories? - if not stat.S_IMODE(mode.st_mode) == 448: - if os.access(dir_, os.W_OK): - os.chmod(dir_, 448) - # Run the extra verification checks - salt.utils.verify.run() - - -def check_user(user, log): - ''' - Check user and assign process uid/gid. - ''' - if 'os' in os.environ: - if os.environ['os'].startswith('Windows'): - return True - if user == getpass.getuser(): - return True - import pwd # after confirming not running Windows - try: - p = pwd.getpwnam(user) - try: - os.setgid(p.pw_gid) - os.setuid(p.pw_uid) - except OSError: - if user == 'root': - msg = 'Sorry, the salt must run as root. http://xkcd.com/838' - else: - msg = 'Salt must be run from root or user "{0}"'.format(user) - log.critical(msg) - return False - except KeyError: - msg = 'User not found: "{0}"'.format(user) - log.critical(msg) - return False - return True - class Master(object): ''' Creates a master server From 6db8dcd0a5a004181d8b7ed46f4edaf9fa5279af Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Tue, 13 Mar 2012 11:47:01 +0000 Subject: [PATCH 090/769] Revert "Add all master processes to pidfile" This reverts commit 5669781770116992f76c20a5a30cbc1ca0d35850. Conflicts: salt/__init__.py --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fd05304..ec4fac5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -29,7 +29,8 @@ def __init__(self): # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] - # Send the pidfile location to the opts + + # Send the pidfile location to the opts if self.cli['pidfile']: self.opts['pidfile'] = self.cli['pidfile'] From 75f88736089cc4786879fa3e7c2d078e0de221af Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Wed, 14 Mar 2012 11:41:11 +0000 Subject: [PATCH 091/769] If we switch to another user pidfiles must be written from original one (possibly root) as pidfiles in /var/run are usually owned by this original user. --- src/saltext/consul/__init__.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ec4fac5..61b98e3 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -98,14 +98,14 @@ def start(self): import logging log = logging.getLogger(__name__) # Late import so logging works correctly + import salt.master + master = salt.master.Master(self.opts) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.opts['pidfile']) if check_user(self.opts['user'], log): - import salt.master - master = salt.master.Master(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.opts['pidfile']) try: master.start() except salt.master.MasterExit: @@ -187,13 +187,13 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) minion = salt.minion.Minion(self.opts) minion.tune_in() except KeyboardInterrupt: @@ -304,14 +304,14 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) + if self.cli['daemon']: + # Late import so logging works correctly + import salt.utils + salt.utils.daemonize() + set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: syndic = salt.minion.Syndic(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) syndic.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Syndic Minion') From 633248c09856379a49ca3e76491d8c4335c4057f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 24 Mar 2012 11:51:24 -0600 Subject: [PATCH 092/769] fix issue with syndic resolution of master 1p --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 61b98e3..bce0065 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -223,7 +223,7 @@ def __prep_opts(self): # Some of the opts need to be changed to match the needed opts # in the minion class. opts['master'] = opts['syndic_master'] - opts['master_ip'] = salt.config.dns_check(opts['master']) + opts['master_ip'] = salt.utils.dns_check(opts['master']) opts['master_uri'] = ('tcp://' + opts['master_ip'] + ':' + str(opts['master_port'])) From f634b892f4873db355995f21b384f502bb3ddab6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 26 Mar 2012 13:53:11 -0600 Subject: [PATCH 093/769] Fix #1002 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index bce0065..fbb8201 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -187,6 +187,7 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) + minion = salt.minion.Minion(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils @@ -194,7 +195,6 @@ def start(self): set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: - minion = salt.minion.Minion(self.opts) minion.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Minion') From e4749bbb221924387986593e7ea5f4f1aebfa3e5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 20 Apr 2012 12:13:02 -0700 Subject: [PATCH 094/769] Clean up code for 3.2 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fbb8201..99e215f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -15,7 +15,7 @@ from salt.utils.process import set_pidfile from salt.utils.verify import check_user, verify_env except ImportError as e: - if e.message != 'No module named _msgpack': + if e.args[0] != 'No module named _msgpack': raise From eff600167d1061e2775f25b9540d2842efae0833 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 26 Apr 2012 16:27:36 -0600 Subject: [PATCH 095/769] move the minion daemonization back to before auth, like it was in 0.9.8 Fix #1145 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 99e215f..c7ddbc5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -187,11 +187,11 @@ def start(self): # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) - minion = salt.minion.Minion(self.opts) if self.cli['daemon']: # Late import so logging works correctly import salt.utils salt.utils.daemonize() + minion = salt.minion.Minion(self.opts) set_pidfile(self.cli['pidfile']) if check_user(self.opts['user'], log): try: From 22c838686b45e4af2ba699e80f065a3694ca8f41 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 26 Apr 2012 16:38:31 -0600 Subject: [PATCH 096/769] Add comment on when the minion is daemonized --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c7ddbc5..954484e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -190,6 +190,10 @@ def start(self): if self.cli['daemon']: # Late import so logging works correctly import salt.utils + # If the minion key has not been accepted, then Salt enters a loop + # waiting for it, if we daemonize later then the minion cound halt + # the boot process waiting for a key to be accepted on the master. + # This is the latest safe place to daemonize salt.utils.daemonize() minion = salt.minion.Minion(self.opts) set_pidfile(self.cli['pidfile']) From af37a923374ee469a7746ae1cc9462fd54098339 Mon Sep 17 00:00:00 2001 From: Baiju Muthukadan Date: Fri, 4 May 2012 20:10:32 +0530 Subject: [PATCH 097/769] replace tab with space and trailing whitespace fix --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 954484e..95b0df6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -29,8 +29,8 @@ def __init__(self): # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] - - # Send the pidfile location to the opts + + # Send the pidfile location to the opts if self.cli['pidfile']: self.opts['pidfile'] = self.cli['pidfile'] From 2134f62bb2ae797f0967ca8f3e35b1e654d4b5e9 Mon Sep 17 00:00:00 2001 From: "L.C. Rees" Date: Thu, 31 May 2012 23:31:27 -0600 Subject: [PATCH 098/769] - py3k updates --- src/saltext/consul/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 95b0df6..db9429d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -62,7 +62,7 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=salt.log.LOG_LEVELS.keys(), + choices=list(salt.log.LOG_LEVELS.keys()), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) @@ -93,7 +93,7 @@ def start(self): salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] ) - for name, level in self.opts['log_granular_levels'].iteritems(): + for name, level in self.opts['log_granular_levels'].items(): salt.log.set_logger_level(name, level) import logging log = logging.getLogger(__name__) @@ -153,10 +153,10 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=salt.log.LOG_LEVELS.keys(), + choices=list(salt.log.LOG_LEVELS.keys()), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) + ', '.join([repr(l) for l in list(salt.log.LOG_LEVELS.keys())])) options, args = parser.parse_args() log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' @@ -181,7 +181,7 @@ def start(self): salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] ) - for name, level in self.opts['log_granular_levels'].iteritems(): + for name, level in self.opts['log_granular_levels'].items(): salt.log.set_logger_level(name, level) import logging # Late import so logging works correctly @@ -272,7 +272,7 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=salt.log.LOG_LEVELS.keys(), + choices=list(salt.log.LOG_LEVELS.keys()), help=('Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) @@ -300,7 +300,7 @@ def start(self): salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] ) - for name, level in self.opts['log_granular_levels'].iteritems(): + for name, level in self.opts['log_granular_levels'].items(): salt.log.set_logger_level(name, level) import logging From c435a307c1ccac61de718ca60854040a54b0ce89 Mon Sep 17 00:00:00 2001 From: "L.C. Rees" Date: Fri, 1 Jun 2012 12:03:17 -0600 Subject: [PATCH 099/769] - exploit how 'iterkeys' is default iterator for dict - use values() - pep8 --- src/saltext/consul/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index db9429d..2d670d8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -62,10 +62,10 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=list(salt.log.LOG_LEVELS.keys()), + choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()]) + ', '.join([repr(l) for l in salt.log.LOG_LEVELS]) ) log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' options, args = parser.parse_args() @@ -153,10 +153,10 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=list(salt.log.LOG_LEVELS.keys()), + choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in list(salt.log.LOG_LEVELS.keys())])) + ', '.join([repr(l) for l in list(salt.log.LOG_LEVELS)])) options, args = parser.parse_args() log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' @@ -272,10 +272,10 @@ def __parse_cli(self): '--log-level', dest='log_level', default='warning', - choices=list(salt.log.LOG_LEVELS.keys()), + choices=list(salt.log.LOG_LEVELS), help=('Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS.keys()])) + ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) ) options, args = parser.parse_args() From 63143fac7a4b9e10ecf5c8361c83966f2bdbd50e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 2 Jun 2012 22:33:56 -0600 Subject: [PATCH 100/769] Pass in user to verify_env --- src/saltext/consul/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 2d670d8..5594963 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -88,7 +88,8 @@ def start(self): os.path.join(self.opts['cachedir'], 'jobs'), os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], - ]) + ], + self.opts['user']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] @@ -176,7 +177,8 @@ def start(self): self.opts['cachedir'], self.opts['extension_modules'], os.path.dirname(self.opts['log_file']), - ]) + ], + self.opts['user']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] @@ -295,7 +297,8 @@ def start(self): ''' verify_env([self.opts['pki_dir'], self.opts['cachedir'], os.path.dirname(self.opts['log_file']), - ]) + ], + self.opts['user']) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] From 65a40d370381f4bd0fe62786678ac2069a24b2de Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 2 Jun 2012 23:13:09 -0600 Subject: [PATCH 101/769] Add more directories to verify ownership on --- src/saltext/consul/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5594963..a4c0b44 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -82,9 +82,11 @@ def start(self): ''' Run the sequence to start a salt master server ''' - verify_env([os.path.join(self.opts['pki_dir'], 'minions'), + verify_env([self.opts['pki_dir'], + os.path.join(self.opts['pki_dir'], 'minions'), os.path.join(self.opts['pki_dir'], 'minions_pre'), os.path.join(self.opts['pki_dir'], 'minions_rejected'), + self.opts['cachedir'], os.path.join(self.opts['cachedir'], 'jobs'), os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], From bdd60a5323b8cde5196fd9328a9b3f2bd0c7711a Mon Sep 17 00:00:00 2001 From: Gavrie Philipson Date: Mon, 18 Jun 2012 19:06:47 +0300 Subject: [PATCH 102/769] Added missing log_format for Syndic. Cleaned up duplications. --- src/saltext/consul/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a4c0b44..35883ba 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -18,6 +18,7 @@ if e.args[0] != 'No module named _msgpack': raise +log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' class Master(object): ''' @@ -65,9 +66,8 @@ def __parse_cli(self): choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS]) - ) - log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' + ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) + options, args = parser.parse_args() salt.log.setup_console_logger(options.log_level, log_format=log_format) @@ -159,11 +159,11 @@ def __parse_cli(self): choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in list(salt.log.LOG_LEVELS)])) + ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' salt.log.setup_console_logger(options.log_level, log_format=log_format) + cli = {'daemon': options.daemon, 'config': options.config, 'user': options.user, @@ -277,13 +277,12 @@ def __parse_cli(self): dest='log_level', default='warning', choices=list(salt.log.LOG_LEVELS), - help=('Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) - ) + help='Console log level. One of %s. For the logfile settings ' + 'see the config file. Default: \'%%default\'.' % + ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level) + salt.log.setup_console_logger(options.log_level, log_format=log_format) cli = {'daemon': options.daemon, 'minion_config': options.minion_config, From 91c592d2b65f03d89e575d927ffe68e2fcfc1e8f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 27 Jun 2012 10:27:32 -0600 Subject: [PATCH 103/769] Hook verify_socket function into the master startup Fix #1452 --- src/saltext/consul/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 35883ba..dd88d2f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,7 +13,7 @@ try: import salt.config from salt.utils.process import set_pidfile - from salt.utils.verify import check_user, verify_env + from salt.utils.verify import check_user, verify_env, verify_socket except ImportError as e: if e.args[0] != 'No module named _msgpack': raise @@ -101,6 +101,14 @@ def start(self): import logging log = logging.getLogger(__name__) # Late import so logging works correctly + if not verify_socket( + self.opts['interface'], + self.opts['publish_port'], + self.opts['ret_port'] + ): + log.critical('The ports are not available to bind') + sys.exit(4) + import salt.master master = salt.master.Master(self.opts) if self.cli['daemon']: From 97da430e265f9a897d48f72a89c44eef9d171aa4 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 30 Jun 2012 23:40:03 +0100 Subject: [PATCH 104/769] Improved logging configuration options from the configuration files. * Added the ability to configure the log levels for the console logger separate from the logfile logger. The log level provided from the cli script will override the console log level set on the configuration file but not the logfile logger setting, unless theres no setting for the logfile setting in which case it will be the same as the console log level. * Added the ability to provide the logging formats for both the console and the logfile. * Added the ability to provide the date format to be used. * The module name formatting("%(name)s") now remembers the widest name, this will improve log readability. * Also catch `KeyboardInterrupt`s while the minion is waiting to connect to the master and not just on `minion.tune_in()`. All these new configuration settings are documented on the configuration file templates. --- src/saltext/consul/__init__.py | 79 +++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 35883ba..1dd476b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -18,15 +18,12 @@ if e.args[0] != 'No module named _msgpack': raise -log_format = '%(asctime)s,%(msecs)03.0f [%(name)-15s][%(levelname)-8s] %(message)s' - class Master(object): ''' Creates a master server ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.config.master_config(self.cli['config']) # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] @@ -62,14 +59,23 @@ def __parse_cli(self): parser.add_option('-l', '--log-level', dest='log_level', - default='warning', choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % + 'see the config file. Default: \'warning\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level, log_format=log_format) + + self.opts = salt.config.master_config(options.config) + + if not options.log_level: + options.log_level = self.opts['log_level'] + + salt.log.setup_console_logger( + options.log_level, + log_format=self.opts['log_fmt_console'], + date_format=self.opts['log_datefmt'] + ) cli = {'daemon': options.daemon, 'config': options.config, @@ -92,12 +98,17 @@ def start(self): self.opts['sock_dir'], ], self.opts['user']) + import salt.log salt.log.setup_logfile_logger( - self.opts['log_file'], self.opts['log_level'] + self.opts['log_file'], + self.opts['log_level_logfile'] or self.opts['log_level'], + log_format=self.opts['log_fmt_logfile'], + date_format=self.opts['log_datefmt'] ) for name, level in self.opts['log_granular_levels'].items(): salt.log.set_logger_level(name, level) + import logging log = logging.getLogger(__name__) # Late import so logging works correctly @@ -121,7 +132,6 @@ class Minion(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = salt.config.minion_config(self.cli['config']) # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] @@ -155,14 +165,23 @@ def __parse_cli(self): parser.add_option('-l', '--log-level', dest='log_level', - default='warning', choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % + 'see the config file. Default: \'warning\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level, log_format=log_format) + + self.opts = salt.config.minion_config(options.config) + + if not options.log_level: + options.log_level = self.opts['log_level'] + + salt.log.setup_console_logger( + options.log_level, + log_format=self.opts['log_fmt_console'], + date_format=self.opts['log_datefmt'] + ) cli = {'daemon': options.daemon, 'config': options.config, @@ -181,12 +200,17 @@ def start(self): os.path.dirname(self.opts['log_file']), ], self.opts['user']) + import salt.log salt.log.setup_logfile_logger( - self.opts['log_file'], self.opts['log_level'] + self.opts['log_file'], + self.opts['log_level_logfile'] or self.opts['log_level'], + log_format=self.opts['log_fmt_logfile'], + date_format=self.opts['log_datefmt'] ) for name, level in self.opts['log_granular_levels'].items(): salt.log.set_logger_level(name, level) + import logging # Late import so logging works correctly import salt.minion @@ -195,18 +219,18 @@ def start(self): # Late import so logging works correctly import salt.utils # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion cound halt + # waiting for it, if we daemonize later then the minion could halt # the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize salt.utils.daemonize() - minion = salt.minion.Minion(self.opts) - set_pidfile(self.cli['pidfile']) - if check_user(self.opts['user'], log): - try: + try: + minion = salt.minion.Minion(self.opts) + set_pidfile(self.cli['pidfile']) + if check_user(self.opts['user'], log): minion.tune_in() - except KeyboardInterrupt: - log.warn('Stopping the Salt Minion') - raise SystemExit('\nExiting on Ctrl-c') + except KeyboardInterrupt: + log.warn('Stopping the Salt Minion') + raise SystemExit('\nExiting on Ctrl-c') class Syndic(object): @@ -215,7 +239,6 @@ class Syndic(object): ''' def __init__(self): self.cli = self.__parse_cli() - self.opts = self.__prep_opts() # command line overrides config if self.cli['user']: self.opts['user'] = self.cli['user'] @@ -275,14 +298,22 @@ def __parse_cli(self): parser.add_option('-l', '--log-level', dest='log_level', - default='warning', choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'%%default\'.' % + 'see the config file. Default: \'warning\'.' % ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - salt.log.setup_console_logger(options.log_level, log_format=log_format) + self.opts = self.__prep_opts() + + if not options.log_level: + options.log_level = self.opts['log_level'] + + salt.log.setup_console_logger( + options.log_level, + log_format=self.opts['log_fmt_console'], + date_format=self.opts['log_datefmt'] + ) cli = {'daemon': options.daemon, 'minion_config': options.minion_config, From 12dfe1b421347697a02e9f733bddd8a926fad1f2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 1 Jul 2012 11:47:49 +0100 Subject: [PATCH 105/769] `OSError`'s fail gracefully when starting up. --- src/saltext/consul/__init__.py | 47 ++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 68f8a9a..62b3c92 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -88,16 +88,19 @@ def start(self): ''' Run the sequence to start a salt master server ''' - verify_env([self.opts['pki_dir'], - os.path.join(self.opts['pki_dir'], 'minions'), - os.path.join(self.opts['pki_dir'], 'minions_pre'), - os.path.join(self.opts['pki_dir'], 'minions_rejected'), - self.opts['cachedir'], - os.path.join(self.opts['cachedir'], 'jobs'), - os.path.dirname(self.opts['log_file']), - self.opts['sock_dir'], - ], - self.opts['user']) + try: + verify_env([ + self.opts['pki_dir'], + os.path.join(self.opts['pki_dir'], 'minions'), + os.path.join(self.opts['pki_dir'], 'minions_pre'), + os.path.join(self.opts['pki_dir'], 'minions_rejected'), + self.opts['cachedir'], + os.path.join(self.opts['cachedir'], 'jobs'), + os.path.dirname(self.opts['log_file']), + self.opts['sock_dir'], + ], self.opts['user']) + except OSError, err: + sys.exit(err.errno) import salt.log salt.log.setup_logfile_logger( @@ -202,12 +205,15 @@ def start(self): ''' Execute this method to start up a minion. ''' - verify_env([self.opts['pki_dir'], - self.opts['cachedir'], - self.opts['extension_modules'], - os.path.dirname(self.opts['log_file']), - ], - self.opts['user']) + try: + verify_env([ + self.opts['pki_dir'], + self.opts['cachedir'], + self.opts['extension_modules'], + os.path.dirname(self.opts['log_file']), + ], self.opts['user']) + except OSError, err: + sys.exit(err.errno) import salt.log salt.log.setup_logfile_logger( @@ -335,10 +341,13 @@ def start(self): ''' Execute this method to start up a syndic. ''' - verify_env([self.opts['pki_dir'], self.opts['cachedir'], + try: + verify_env([ + self.opts['pki_dir'], self.opts['cachedir'], os.path.dirname(self.opts['log_file']), - ], - self.opts['user']) + ], self.opts['user']) + except OSError, err: + sys.exit(err.errno) import salt.log salt.log.setup_logfile_logger( self.opts['log_file'], self.opts['log_level'] From 51effc6d4cd9c34d116b12bdc451243f6181d46e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 6 Jul 2012 01:19:08 -0600 Subject: [PATCH 106/769] Add the sock_dir to the verify_env on the minion --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 62b3c92..87ce2ae 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -209,6 +209,7 @@ def start(self): verify_env([ self.opts['pki_dir'], self.opts['cachedir'], + self.opts['sock_dir'], self.opts['extension_modules'], os.path.dirname(self.opts['log_file']), ], self.opts['user']) From 8911ab76b3c013363d8f548119903e1b3f0dbbc4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 7 Jul 2012 22:43:43 -0600 Subject: [PATCH 107/769] Move pillar into module dir --- src/saltext/consul/pillar/__init__.py | 373 ++++++++++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 src/saltext/consul/pillar/__init__.py diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py new file mode 100644 index 0000000..b4451af --- /dev/null +++ b/src/saltext/consul/pillar/__init__.py @@ -0,0 +1,373 @@ +''' +Render the pillar data +''' + +# Import python libs +import os +import copy +import collections +import logging +import subprocess + +# Import Salt libs +import salt.loader +import salt.fileclient +import salt.minion +import salt.crypt +from salt._compat import string_types +from salt.template import compile_template + +# Import third party libs +import zmq +import yaml + +log = logging.getLogger(__name__) + + +def hiera(conf, grains=None): + ''' + Execute hiera and return the data + ''' + if not isinstance(grains, dict): + grains = {} + cmd = 'hiera {0}'.format(conf) + for key, val in grains.items(): + if isinstance(val, string_types): + cmd += ' {0}={1}'.format(key, val) + out = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + shell=True + ).communicate()[0] + return yaml.safe_load(out) + + +def cmd_yaml(command, grains=None): + ''' + Execute a command and read the output as YAML + ''' + out = subprocess.Popen( + command, + stdout=subprocess.PIPE, + shell=True + ).communicate()[0] + return yaml.safe_load(out) + + +ext_pillar = {'hiera': hiera, + 'cmd_yaml': cmd_yaml} + + +def get_pillar(opts, grains, id_, env=None): + ''' + Return the correct pillar driver based on the file_client option + ''' + try: + return { + 'remote': RemotePillar, + 'local': Pillar + }.get(opts['file_client'], 'local')(opts, grains, id_, env) + except KeyError: + return Pillar(opts, grains, id_, env) + + +class RemotePillar(object): + ''' + Get the pillar from the master + ''' + def __init__(self, opts, grains, id_, env): + self.opts = opts + self.opts['environment'] = env + self.grains = grains + self.id_ = id_ + self.serial = salt.payload.Serial(self.opts) + self.sreq = salt.payload.SREQ(self.opts['master_uri']) + self.auth = salt.crypt.SAuth(opts) + + def compile_pillar(self): + ''' + Return the pillar data from the master + ''' + load = {'id': self.id_, + 'grains': self.grains, + 'env': self.opts['environment'], + 'cmd': '_pillar'} + return self.auth.crypticle.loads( + self.sreq.send('aes', self.auth.crypticle.dumps(load), 3, 7200) + ) + + + +class Pillar(object): + ''' + Read over the pillar top files and render the pillar data + ''' + def __init__(self, opts, grains, id_, env): + # use the local file client + self.opts = self.__gen_opts(opts, grains, id_, env) + self.client = salt.fileclient.get_file_client(self.opts) + self.matcher = salt.minion.Matcher(self.opts) + self.rend = salt.loader.render(self.opts, {}) + + def __gen_opts(self, opts, grains, id_, env=None): + ''' + The options need to be altered to conform to the file client + ''' + opts = copy.deepcopy(opts) + opts['file_roots'] = opts['pillar_roots'] + opts['file_client'] = 'local' + opts['grains'] = grains + opts['id'] = id_ + if 'environment' not in opts: + opts['environment'] = env + if opts['state_top'].startswith('salt://'): + opts['state_top'] = opts['state_top'] + elif opts['state_top'].startswith('/'): + opts['state_top'] = os.path.join('salt://', opts['state_top'][1:]) + else: + opts['state_top'] = os.path.join('salt://', opts['state_top']) + return opts + + def _get_envs(self): + ''' + Pull the file server environments out of the master options + ''' + envs = set(['base']) + if 'file_roots' in self.opts: + envs.update(list(self.opts['file_roots'])) + return envs + + def get_tops(self): + ''' + Gather the top files + ''' + tops = collections.defaultdict(list) + include = collections.defaultdict(list) + done = collections.defaultdict(list) + # Gather initial top files + if self.opts['environment']: + tops[self.opts['environment']] = [ + compile_template( + self.client.cache_file( + self.opts['state_top'], + self.opts['environment'] + ), + self.rend, + self.opts['renderer'], + self.opts['environment'] + ) + ] + else: + for env in self._get_envs(): + tops[env].append( + compile_template( + self.client.cache_file( + self.opts['state_top'], + env + ), + self.rend, + self.opts['renderer'], + env=env + ) + ) + + # Search initial top files for includes + for env, ctops in tops.items(): + for ctop in ctops: + if not 'include' in ctop: + continue + for sls in ctop['include']: + include[env].append(sls) + ctop.pop('include') + # Go through the includes and pull out the extra tops and add them + while include: + pops = [] + for env, states in include.items(): + pops.append(env) + if not states: + continue + for sls in states: + if sls in done[env]: + continue + tops[env].append( + compile_template( + self.client.get_state( + sls, + env + ), + self.rend, + self.opts['renderer'], + env=env + ) + ) + done[env].append(sls) + for env in pops: + if env in include: + include.pop(env) + + return tops + + def merge_tops(self, tops): + ''' + Cleanly merge the top files + ''' + top = collections.defaultdict(dict) + for sourceenv, ctops in tops.items(): + for ctop in ctops: + for env, targets in ctop.items(): + if env == 'include': + continue + for tgt in targets: + if not tgt in top[env]: + top[env][tgt] = ctop[env][tgt] + continue + matches = [] + states = set() + for comp in top[env][tgt]: + if isinstance(comp, dict): + matches.append(comp) + if isinstance(comp, string_types): + states.add(comp) + top[env][tgt] = matches + top[env][tgt].extend(list(states)) + return top + + def get_top(self): + ''' + Returns the high data derived from the top file + ''' + tops = self.get_tops() + return self.merge_tops(tops) + + def top_matches(self, top): + ''' + Search through the top high data for matches and return the states + that this minion needs to execute. + + Returns: + {'env': ['state1', 'state2', ...]} + ''' + matches = {} + for env, body in top.items(): + if self.opts['environment']: + if not env == self.opts['environment']: + continue + for match, data in body.items(): + if self.matcher.confirm_top( + match, + data, + self.opts.get('nodegroups', {}), + ): + if env not in matches: + matches[env] = [] + for item in data: + if isinstance(item, string_types): + matches[env].append(item) + ext_matches = self.client.ext_nodes() + for env in ext_matches: + if env in matches: + matches[env] = list(set(ext_matches[env]).union(matches[env])) + else: + matches[env] = ext_matches[env] + return matches + + def render_pstate(self, sls, env, mods): + ''' + Collect a single pillar sls file and render it + ''' + err = '' + errors = [] + fn_ = self.client.get_state(sls, env) + if not fn_: + errors.append(('Specified SLS {0} in environment {1} is not' + ' available on the salt master').format(sls, env)) + state = None + try: + state = compile_template( + fn_, self.rend, self.opts['renderer'], env, sls) + except Exception as exc: + errors.append(('Rendering SLS {0} failed, render error:\n{1}' + .format(sls, exc))) + mods.add(sls) + nstate = None + if state: + if not isinstance(state, dict): + errors.append(('SLS {0} does not render to a dictionary' + .format(sls))) + else: + if 'include' in state: + if not isinstance(state['include'], list): + err = ('Include Declaration in SLS {0} is not formed ' + 'as a list'.format(sls)) + errors.append(err) + else: + for sub_sls in state.pop('include'): + if sub_sls not in mods: + nstate, mods, err = self.render_pstate( + sub_sls, + env, + mods + ) + if nstate: + state.update(nstate) + if err: + errors += err + return state, mods, errors + + def render_pillar(self, matches): + ''' + Extract the sls pillar files from the matches and render them into the + pillar + ''' + pillar = {} + errors = [] + for env, pstates in matches.items(): + mods = set() + for sls in pstates: + pstate, mods, err = self.render_pstate(sls, env, mods) + if pstate: + pillar.update(pstate) + if err: + errors += err + return pillar, errors + + def ext_pillar(self): + ''' + Render the external pillar data + ''' + if not 'ext_pillar' in self.opts: + return {} + if not isinstance(self.opts['ext_pillar'], list): + log.critical('The "ext_pillar" option is malformed') + return {} + ext = {} + for run in self.opts['ext_pillar']: + if not isinstance(run, dict): + log.critical('The "ext_pillar" option is malformed') + return {} + if len(run) != 1: + log.critical('The "ext_pillar" option is malformed') + return {} + for key, val in run.items(): + if key not in ext_pillar: + err = ('Specified ext_pillar interface {0} is ' + 'unavailable').format(key) + log.critical(err) + return {} + try: + ext.update(ext_pillar[key](val, self.opts['grains'])) + except Exception as e: + log.critical('Failed to load ext_pillar {0}'.format(key)) + return ext + + def compile_pillar(self): + ''' + Render the pillar dta and return + ''' + top = self.get_top() + matches = self.top_matches(top) + pillar, errors = self.render_pillar(matches) + pillar.update(self.ext_pillar()) + if errors: + return errors + return pillar From 03097d64893468ea934cd3bd4b6e440bd4ac18db Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 7 Jul 2012 23:51:24 -0600 Subject: [PATCH 108/769] Hook ext_pillar method into loader interfaces --- src/saltext/consul/pillar/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b4451af..8f43702 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -108,6 +108,7 @@ def __init__(self, opts, grains, id_, env): self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) self.rend = salt.loader.render(self.opts, {}) + self.ext_pillars = salt.loader.pillars(self.opts, grains) def __gen_opts(self, opts, grains, id_, env=None): ''' @@ -349,13 +350,18 @@ def ext_pillar(self): log.critical('The "ext_pillar" option is malformed') return {} for key, val in run.items(): - if key not in ext_pillar: + if key not in self.pillars: err = ('Specified ext_pillar interface {0} is ' 'unavailable').format(key) log.critical(err) - return {} + continue try: - ext.update(ext_pillar[key](val, self.opts['grains'])) + if isinstance(val, dict): + ext.update(self.pillars[key](**val)) + elif isinstance(val, list): + ext.update(self.pillars[key](*val)) + else: + ext.update(self.pillars[key](val)) except Exception as e: log.critical('Failed to load ext_pillar {0}'.format(key)) return ext From 7c3624f5677a3e1d7af53cf9f34fde428121fea9 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 8 Jul 2012 00:09:07 -0600 Subject: [PATCH 109/769] fix ext_pillars reference --- src/saltext/consul/pillar/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8f43702..f82389a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -108,7 +108,7 @@ def __init__(self, opts, grains, id_, env): self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) self.rend = salt.loader.render(self.opts, {}) - self.ext_pillars = salt.loader.pillars(self.opts, grains) + self.ext_pillars = salt.loader.pillars(self.opts) def __gen_opts(self, opts, grains, id_, env=None): ''' @@ -336,6 +336,7 @@ def ext_pillar(self): ''' Render the external pillar data ''' + print self.ext_pillars.keys() if not 'ext_pillar' in self.opts: return {} if not isinstance(self.opts['ext_pillar'], list): @@ -346,22 +347,19 @@ def ext_pillar(self): if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} - if len(run) != 1: - log.critical('The "ext_pillar" option is malformed') - return {} for key, val in run.items(): - if key not in self.pillars: + if key not in self.ext_pillars: err = ('Specified ext_pillar interface {0} is ' 'unavailable').format(key) log.critical(err) continue try: if isinstance(val, dict): - ext.update(self.pillars[key](**val)) + ext.update(self.ext_pillars[key](**val)) elif isinstance(val, list): - ext.update(self.pillars[key](*val)) + ext.update(self.ext_pillars[key](*val)) else: - ext.update(self.pillars[key](val)) + ext.update(self.ext_pillars[key](val)) except Exception as e: log.critical('Failed to load ext_pillar {0}'.format(key)) return ext From 8f6387a1e26af07170d0286175d2cdf4e6a0bf6c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 8 Jul 2012 00:11:03 -0600 Subject: [PATCH 110/769] kill stray print statement --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f82389a..a36fa30 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -336,7 +336,6 @@ def ext_pillar(self): ''' Render the external pillar data ''' - print self.ext_pillars.keys() if not 'ext_pillar' in self.opts: return {} if not isinstance(self.opts['ext_pillar'], list): From a32b38c0945a974b2ff6cbd173627a31a3920bc2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 8 Jul 2012 00:38:06 -0600 Subject: [PATCH 111/769] Pack the module functions into ext_pillar modules --- src/saltext/consul/pillar/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a36fa30..4eae974 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -107,8 +107,9 @@ def __init__(self, opts, grains, id_, env): self.opts = self.__gen_opts(opts, grains, id_, env) self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) - self.rend = salt.loader.render(self.opts, {}) - self.ext_pillars = salt.loader.pillars(self.opts) + self.functions = salt.loader.minion_mods(self.opts) + self.rend = salt.loader.render(self.opts, self.functions) + self.ext_pillars = salt.loader.pillars(self.opts, self.functions) def __gen_opts(self, opts, grains, id_, env=None): ''' From 67edc31530965b7374ed36ad5dd17421b48e9af0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 8 Jul 2012 00:44:33 -0600 Subject: [PATCH 112/769] detatch old ext_pillar functions --- src/saltext/consul/pillar/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4eae974..c301480 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -42,22 +42,6 @@ def hiera(conf, grains=None): return yaml.safe_load(out) -def cmd_yaml(command, grains=None): - ''' - Execute a command and read the output as YAML - ''' - out = subprocess.Popen( - command, - stdout=subprocess.PIPE, - shell=True - ).communicate()[0] - return yaml.safe_load(out) - - -ext_pillar = {'hiera': hiera, - 'cmd_yaml': cmd_yaml} - - def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option From 37b44a7fb4e0f12b5b901e18a74f8fcbf9779f20 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 8 Jul 2012 00:56:30 -0600 Subject: [PATCH 113/769] Remove deprecated hiera function --- src/saltext/consul/pillar/__init__.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c301480..fc2adf6 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -24,24 +24,6 @@ log = logging.getLogger(__name__) -def hiera(conf, grains=None): - ''' - Execute hiera and return the data - ''' - if not isinstance(grains, dict): - grains = {} - cmd = 'hiera {0}'.format(conf) - for key, val in grains.items(): - if isinstance(val, string_types): - cmd += ' {0}={1}'.format(key, val) - out = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - shell=True - ).communicate()[0] - return yaml.safe_load(out) - - def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option From e6b6cfd8e4e3736b1d3b921ab92b2646795fa0ae Mon Sep 17 00:00:00 2001 From: "E.G. Galano" Date: Tue, 10 Jul 2012 18:31:28 -0700 Subject: [PATCH 114/769] Fixes syndic init failure due to cli attribute error introduced in commit: 97da430e265f9a897d48f72a89c44eef9d171aa4 --- src/saltext/consul/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 87ce2ae..e1be4b1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -258,13 +258,13 @@ def __init__(self): if self.cli['user']: self.opts['user'] = self.cli['user'] - def __prep_opts(self): + def __prep_opts(self, cli): ''' Generate the opts used by the syndic ''' - opts = salt.config.master_config(self.cli['master_config']) + opts = salt.config.master_config(cli['master_config']) opts['_minion_conf_file'] = opts['conf_file'] - opts.update(salt.config.minion_config(self.cli['minion_config'])) + opts.update(salt.config.minion_config(cli['minion_config'])) if 'syndic_master' in opts: # Some of the opts need to be changed to match the needed opts # in the minion class. @@ -319,7 +319,14 @@ def __parse_cli(self): ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) options, args = parser.parse_args() - self.opts = self.__prep_opts() + + cli = {'daemon': options.daemon, + 'minion_config': options.minion_config, + 'master_config': options.master_config, + 'pidfile': options.pidfile, + 'user': options.user} + + self.opts = self.__prep_opts(cli) if not options.log_level: options.log_level = self.opts['log_level'] @@ -330,12 +337,6 @@ def __parse_cli(self): date_format=self.opts['log_datefmt'] ) - cli = {'daemon': options.daemon, - 'minion_config': options.minion_config, - 'master_config': options.master_config, - 'pidfile': options.pidfile, - 'user': options.user} - return cli def start(self): From baafa9a8de33e99f18de0364c2ad67ec3f491ee0 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 14 Jul 2012 23:58:22 -0600 Subject: [PATCH 115/769] Add error detection to pillar top files --- src/saltext/consul/pillar/__init__.py | 74 ++++++++++++++++----------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fc2adf6..3e15224 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -112,32 +112,38 @@ def get_tops(self): tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) + errors = [] # Gather initial top files - if self.opts['environment']: - tops[self.opts['environment']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['environment'] - ), - self.rend, - self.opts['renderer'], - self.opts['environment'] - ) - ] - else: - for env in self._get_envs(): - tops[env].append( + try: + if self.opts['environment']: + tops[self.opts['environment']] = [ compile_template( self.client.cache_file( self.opts['state_top'], - env + self.opts['environment'] ), self.rend, self.opts['renderer'], - env=env + self.opts['environment'] ) - ) + ] + else: + for env in self._get_envs(): + tops[env].append( + compile_template( + self.client.cache_file( + self.opts['state_top'], + env + ), + self.rend, + self.opts['renderer'], + env=env + ) + ) + except Exception as exc: + errors.append( + ('Rendering Primary Top file failed, render error:\n{0}' + .format(exc))) # Search initial top files for includes for env, ctops in tops.items(): @@ -157,23 +163,28 @@ def get_tops(self): for sls in states: if sls in done[env]: continue - tops[env].append( - compile_template( - self.client.get_state( - sls, - env - ), - self.rend, - self.opts['renderer'], - env=env + try: + tops[env].append( + compile_template( + self.client.get_state( + sls, + env + ), + self.rend, + self.opts['renderer'], + env=env + ) ) - ) + except Exception as exc: + errors.append( + ('Rendering Top file {0} failed, render error' + ':\n{1}').format(sls, exc)) done[env].append(sls) for env in pops: if env in include: include.pop(env) - return tops + return tops, errors def merge_tops(self, tops): ''' @@ -204,7 +215,7 @@ def get_top(self): ''' Returns the high data derived from the top file ''' - tops = self.get_tops() + tops, errors = self.get_tops() return self.merge_tops(tops) def top_matches(self, top): @@ -334,10 +345,11 @@ def compile_pillar(self): ''' Render the pillar dta and return ''' - top = self.get_top() + top, terrors = self.get_top() matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) pillar.update(self.ext_pillar()) + errors.extend(terrors) if errors: return errors return pillar From 7c46497e58be4b9a57d2bf65b2db7593e7337545 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 15 Jul 2012 00:13:12 -0600 Subject: [PATCH 116/769] Fix #1515 --- src/saltext/consul/pillar/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3e15224..51b4193 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -216,7 +216,7 @@ def get_top(self): Returns the high data derived from the top file ''' tops, errors = self.get_tops() - return self.merge_tops(tops) + return self.merge_tops(tops), errors def top_matches(self, top): ''' @@ -351,5 +351,7 @@ def compile_pillar(self): pillar.update(self.ext_pillar()) errors.extend(terrors) if errors: - return errors + for error in errors: + log.critical('Pillar render error: {0}'.format(error)) + return {} return pillar From 28f2713b1e19ac8bbacc6fc22385e011e835a364 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 22 Jul 2012 12:32:36 +0100 Subject: [PATCH 117/769] Log levels sorting on master and minion cli's help output. (Cosmetic) --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e1be4b1..d7b5452 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -62,7 +62,7 @@ def __parse_cli(self): choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'warning\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) + ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES])) options, args = parser.parse_args() @@ -179,7 +179,7 @@ def __parse_cli(self): choices=list(salt.log.LOG_LEVELS), help='Console log level. One of %s. For the logfile settings ' 'see the config file. Default: \'warning\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) + ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES])) options, args = parser.parse_args() From 5aa57cc241796c780e7dd4b1c69c3065263fbcde Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Jul 2012 10:40:19 +0100 Subject: [PATCH 118/769] Dependencies version printing. * This commit allows printing the deps version numbers from salt's cli tools(as long as the optparer used is `salt.utils.parser`). It can be useful in some debugging report situations. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d7b5452..7016c6e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,12 +6,12 @@ # Import python libs import os import sys -import optparse # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail try: import salt.config + from salt.utils import parser as optparse from salt.utils.process import set_pidfile from salt.utils.verify import check_user, verify_env, verify_socket except ImportError as e: From 4d8811fc84fcf532ba84d48ef7b15e5c38ea6795 Mon Sep 17 00:00:00 2001 From: Grier Johnson Date: Tue, 24 Jul 2012 14:55:15 -0700 Subject: [PATCH 119/769] Allowing permissive pki group ownership Allow for an optional directive that allows the salt pki_dir to be owned by any group that root is a member of, not just the primary group listed in the passwd directory. This is potentially insecure, but allows some limited access for headless accounts to run salt without the need for sudo and the config that goes along with that. Conflicts: salt/__init__.py salt/utils/verify.py --- src/saltext/consul/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7016c6e..3090842 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -98,7 +98,8 @@ def start(self): os.path.join(self.opts['cachedir'], 'jobs'), os.path.dirname(self.opts['log_file']), self.opts['sock_dir'], - ], self.opts['user']) + ], self.opts['user'], + permissive=self.opts['permissive_pki_access']) except OSError, err: sys.exit(err.errno) @@ -212,7 +213,8 @@ def start(self): self.opts['sock_dir'], self.opts['extension_modules'], os.path.dirname(self.opts['log_file']), - ], self.opts['user']) + ], self.opts['user'], + permissive=self.opts['permissive_pki_access']) except OSError, err: sys.exit(err.errno) @@ -347,7 +349,8 @@ def start(self): verify_env([ self.opts['pki_dir'], self.opts['cachedir'], os.path.dirname(self.opts['log_file']), - ], self.opts['user']) + ], self.opts['user'], + permissive=self.opts['permissive_pki_access']) except OSError, err: sys.exit(err.errno) import salt.log From dd8a9ef32701f3e9910e2bac0d516d1006947635 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 4 Aug 2012 19:58:32 +0100 Subject: [PATCH 120/769] Parsers(master, minion, syndic) cleanup code. Moved the minion, master and syndic parsers to use mixins to clean up, reuse, and automate some of the required procedures. --- src/saltext/consul/__init__.py | 350 ++++++--------------------------- 1 file changed, 61 insertions(+), 289 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3090842..661dffc 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,7 +1,6 @@ ''' Make me some salt! ''' -from salt.version import __version__ # Import python libs import os @@ -9,370 +8,143 @@ # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail +from salt.version import __version__ + try: import salt.config - from salt.utils import parser as optparse - from salt.utils.process import set_pidfile + from salt.utils import parsers from salt.utils.verify import check_user, verify_env, verify_socket except ImportError as e: if e.args[0] != 'No module named _msgpack': raise -class Master(object): +class Master(parsers.MasterOptionParser): ''' Creates a master server ''' - def __init__(self): - self.cli = self.__parse_cli() - # command line overrides config - if self.cli['user']: - self.opts['user'] = self.cli['user'] - - # Send the pidfile location to the opts - if self.cli['pidfile']: - self.opts['pidfile'] = self.cli['pidfile'] - - def __parse_cli(self): - ''' - Parse the cli for options passed to a master daemon - ''' - import salt.log - parser = optparse.OptionParser(version="%%prog %s" % __version__) - parser.add_option('-d', - '--daemon', - dest='daemon', - default=False, - action='store_true', - help='Run the master as a daemon') - parser.add_option('-c', - '--config', - dest='config', - default='/etc/salt/master', - help='Pass in an alternative configuration file') - parser.add_option('-u', - '--user', - dest='user', - help='Specify user to run master') - parser.add_option('--pid-file', - dest='pidfile', - help=('Specify the location of the pidfile.')) - parser.add_option('-l', - '--log-level', - dest='log_level', - choices=list(salt.log.LOG_LEVELS), - help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'warning\'.' % - ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES])) - - options, args = parser.parse_args() - - self.opts = salt.config.master_config(options.config) - - if not options.log_level: - options.log_level = self.opts['log_level'] - - salt.log.setup_console_logger( - options.log_level, - log_format=self.opts['log_fmt_console'], - date_format=self.opts['log_datefmt'] - ) - - cli = {'daemon': options.daemon, - 'config': options.config, - 'user': options.user, - 'pidfile': options.pidfile} - - return cli def start(self): ''' Run the sequence to start a salt master server ''' + self.parse_args() + try: verify_env([ - self.opts['pki_dir'], - os.path.join(self.opts['pki_dir'], 'minions'), - os.path.join(self.opts['pki_dir'], 'minions_pre'), - os.path.join(self.opts['pki_dir'], 'minions_rejected'), - self.opts['cachedir'], - os.path.join(self.opts['cachedir'], 'jobs'), - os.path.dirname(self.opts['log_file']), - self.opts['sock_dir'], - ], self.opts['user'], - permissive=self.opts['permissive_pki_access']) + self.config['pki_dir'], + os.path.join(self.config['pki_dir'], 'minions'), + os.path.join(self.config['pki_dir'], 'minions_pre'), + os.path.join(self.config['pki_dir'], 'minions_rejected'), + self.config['cachedir'], + os.path.join(self.config['cachedir'], 'jobs'), + os.path.dirname(self.config['log_file']), + self.config['sock_dir'], + ], self.config['user'], + permissive=self.config['permissive_pki_access']) except OSError, err: sys.exit(err.errno) - import salt.log - salt.log.setup_logfile_logger( - self.opts['log_file'], - self.opts['log_level_logfile'] or self.opts['log_level'], - log_format=self.opts['log_fmt_logfile'], - date_format=self.opts['log_datefmt'] - ) - for name, level in self.opts['log_granular_levels'].items(): - salt.log.set_logger_level(name, level) + self.setup_logfile_logger() import logging log = logging.getLogger(__name__) # Late import so logging works correctly - if not verify_socket( - self.opts['interface'], - self.opts['publish_port'], - self.opts['ret_port'] - ): - log.critical('The ports are not available to bind') - sys.exit(4) + if not verify_socket(self.config['interface'], + self.config['publish_port'], + self.config['ret_port']): + self.exit(4, 'The ports are not available to bind') import salt.master - master = salt.master.Master(self.opts) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.opts['pidfile']) - if check_user(self.opts['user'], log): + master = salt.master.Master(self.config) + self.daemonize_if_required() + self.set_pidfile() + if check_user(self.config['user'], log): try: master.start() except salt.master.MasterExit: sys.exit() -class Minion(object): +class Minion(parsers.MinionOptionParser): ''' Create a minion server ''' - def __init__(self): - self.cli = self.__parse_cli() - # command line overrides config - if self.cli['user']: - self.opts['user'] = self.cli['user'] - - def __parse_cli(self): - ''' - Parse the cli input - ''' - import salt.log - parser = optparse.OptionParser(version="%%prog %s" % __version__) - parser.add_option('-d', - '--daemon', - dest='daemon', - default=False, - action='store_true', - help='Run the minion as a daemon') - parser.add_option('-c', - '--config', - dest='config', - default='/etc/salt/minion', - help='Pass in an alternative configuration file') - parser.add_option('-u', - '--user', - dest='user', - help='Specify user to run minion') - parser.add_option('--pid-file', - dest='pidfile', - default='/var/run/salt-minion.pid', - help=('Specify the location of the pidfile. Default' - ' %default')) - parser.add_option('-l', - '--log-level', - dest='log_level', - choices=list(salt.log.LOG_LEVELS), - help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'warning\'.' % - ', '.join([repr(l) for l in salt.log.SORTED_LEVEL_NAMES])) - - options, args = parser.parse_args() - - self.opts = salt.config.minion_config(options.config) - - if not options.log_level: - options.log_level = self.opts['log_level'] - - salt.log.setup_console_logger( - options.log_level, - log_format=self.opts['log_fmt_console'], - date_format=self.opts['log_datefmt'] - ) - - cli = {'daemon': options.daemon, - 'config': options.config, - 'user': options.user, - 'pidfile': options.pidfile} - - return cli def start(self): ''' Execute this method to start up a minion. ''' + self.parse_args() + try: verify_env([ - self.opts['pki_dir'], - self.opts['cachedir'], - self.opts['sock_dir'], - self.opts['extension_modules'], - os.path.dirname(self.opts['log_file']), - ], self.opts['user'], - permissive=self.opts['permissive_pki_access']) + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + os.path.dirname(self.config['log_file']), + ], self.config['user'], + permissive=self.config['permissive_pki_access']) except OSError, err: sys.exit(err.errno) - import salt.log - salt.log.setup_logfile_logger( - self.opts['log_file'], - self.opts['log_level_logfile'] or self.opts['log_level'], - log_format=self.opts['log_fmt_logfile'], - date_format=self.opts['log_datefmt'] - ) - for name, level in self.opts['log_granular_levels'].items(): - salt.log.set_logger_level(name, level) + self.setup_logfile_logger() import logging # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion could halt - # the boot process waiting for a key to be accepted on the master. - # This is the latest safe place to daemonize - salt.utils.daemonize() + # If the minion key has not been accepted, then Salt enters a loop + # waiting for it, if we daemonize later then the minion could halt + # the boot process waiting for a key to be accepted on the master. + # This is the latest safe place to daemonize + self.daemonize_if_required() try: - minion = salt.minion.Minion(self.opts) - set_pidfile(self.cli['pidfile']) - if check_user(self.opts['user'], log): + minion = salt.minion.Minion(self.config) + self.set_pidfile() + if check_user(self.config['user'], log): minion.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Minion') raise SystemExit('\nExiting on Ctrl-c') -class Syndic(object): +class Syndic(parsers.SyndicOptionParser): ''' Create a syndic server ''' - def __init__(self): - self.cli = self.__parse_cli() - # command line overrides config - if self.cli['user']: - self.opts['user'] = self.cli['user'] - - def __prep_opts(self, cli): - ''' - Generate the opts used by the syndic - ''' - opts = salt.config.master_config(cli['master_config']) - opts['_minion_conf_file'] = opts['conf_file'] - opts.update(salt.config.minion_config(cli['minion_config'])) - if 'syndic_master' in opts: - # Some of the opts need to be changed to match the needed opts - # in the minion class. - opts['master'] = opts['syndic_master'] - opts['master_ip'] = salt.utils.dns_check(opts['master']) - - opts['master_uri'] = ('tcp://' + opts['master_ip'] + - ':' + str(opts['master_port'])) - opts['_master_conf_file'] = opts['conf_file'] - opts.pop('conf_file') - return opts - err = ('The syndic_master needs to be configured in the salt master ' - 'config, EXITING!\n') - sys.stderr.write(err) - sys.exit(2) - - def __parse_cli(self): - ''' - Parse the cli for options passed to a syndic daemon - ''' - import salt.log - parser = optparse.OptionParser(version="%%prog %s" % __version__) - parser.add_option('-d', - '--daemon', - dest='daemon', - default=False, - action='store_true', - help='Run the syndic as a daemon') - parser.add_option('--master-config', - dest='master_config', - default='/etc/salt/master', - help='Pass in an alternative master configuration file') - parser.add_option('--minion-config', - dest='minion_config', - default='/etc/salt/minion', - help='Pass in an alternative minion configuration file') - parser.add_option('-u', - '--user', - dest='user', - help='Specify user to run syndic') - parser.add_option('--pid-file', - dest='pidfile', - default='/var/run/salt-syndic.pid', - help=('Specify the location of the pidfile. Default' - ' %default')) - parser.add_option('-l', - '--log-level', - dest='log_level', - choices=list(salt.log.LOG_LEVELS), - help='Console log level. One of %s. For the logfile settings ' - 'see the config file. Default: \'warning\'.' % - ', '.join([repr(l) for l in salt.log.LOG_LEVELS])) - - options, args = parser.parse_args() - - cli = {'daemon': options.daemon, - 'minion_config': options.minion_config, - 'master_config': options.master_config, - 'pidfile': options.pidfile, - 'user': options.user} - - self.opts = self.__prep_opts(cli) - - if not options.log_level: - options.log_level = self.opts['log_level'] - - salt.log.setup_console_logger( - options.log_level, - log_format=self.opts['log_fmt_console'], - date_format=self.opts['log_datefmt'] - ) - - return cli def start(self): ''' Execute this method to start up a syndic. ''' + self.parse_args() try: verify_env([ - self.opts['pki_dir'], self.opts['cachedir'], - os.path.dirname(self.opts['log_file']), - ], self.opts['user'], - permissive=self.opts['permissive_pki_access']) + self.config['pki_dir'], self.config['cachedir'], + os.path.dirname(self.config['log_file']), + ], + self.config['user'], + permissive=self.config['permissive_pki_access'] + ) except OSError, err: sys.exit(err.errno) - import salt.log - salt.log.setup_logfile_logger( - self.opts['log_file'], self.opts['log_level'] - ) - for name, level in self.opts['log_granular_levels'].items(): - salt.log.set_logger_level(name, level) + + + self.setup_logfile_logger() import logging # Late import so logging works correctly import salt.minion log = logging.getLogger(__name__) - if self.cli['daemon']: - # Late import so logging works correctly - import salt.utils - salt.utils.daemonize() - set_pidfile(self.cli['pidfile']) - if check_user(self.opts['user'], log): + + self.daemonize_if_required() + self.set_pidfile() + + if check_user(self.config['user'], log): try: - syndic = salt.minion.Syndic(self.opts) + syndic = salt.minion.Syndic(self.config) syndic.tune_in() except KeyboardInterrupt: log.warn('Stopping the Salt Syndic Minion') From 9d84e8269bc049b85b05f383be78d9b76095fc34 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 10 Aug 2012 11:23:54 +0100 Subject: [PATCH 121/769] Fix saltstack/salt#1756 * Only use logging if the console logging has been properly setup, in case it hasn't been, write the message to `sys.stderr`. * Removed the log argument passed to `salt.verify.check_user`, it's not needed anymore. --- src/saltext/consul/__init__.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 661dffc..adace0f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -5,6 +5,7 @@ # Import python libs import os import sys +import logging # Import salt libs, the try block bypasses an issue at build time so that c # modules don't cause the build to fail @@ -46,8 +47,6 @@ def start(self): self.setup_logfile_logger() - import logging - log = logging.getLogger(__name__) # Late import so logging works correctly if not verify_socket(self.config['interface'], self.config['publish_port'], @@ -58,7 +57,7 @@ def start(self): master = salt.master.Master(self.config) self.daemonize_if_required() self.set_pidfile() - if check_user(self.config['user'], log): + if check_user(self.config['user']): try: master.start() except salt.master.MasterExit: @@ -90,10 +89,8 @@ def start(self): self.setup_logfile_logger() - import logging # Late import so logging works correctly import salt.minion - log = logging.getLogger(__name__) # If the minion key has not been accepted, then Salt enters a loop # waiting for it, if we daemonize later then the minion could halt # the boot process waiting for a key to be accepted on the master. @@ -102,10 +99,10 @@ def start(self): try: minion = salt.minion.Minion(self.config) self.set_pidfile() - if check_user(self.config['user'], log): + if check_user(self.config['user']): minion.tune_in() except KeyboardInterrupt: - log.warn('Stopping the Salt Minion') + logging.getLogger(__name__).warn('Stopping the Salt Minion') raise SystemExit('\nExiting on Ctrl-c') @@ -133,19 +130,17 @@ def start(self): self.setup_logfile_logger() - import logging - # Late import so logging works correctly import salt.minion - log = logging.getLogger(__name__) - self.daemonize_if_required() self.set_pidfile() - if check_user(self.config['user'], log): + if check_user(self.config['user']): try: syndic = salt.minion.Syndic(self.config) syndic.tune_in() except KeyboardInterrupt: - log.warn('Stopping the Salt Syndic Minion') + logging.getLogger(__name__).warn( + 'Stopping the Salt Syndic Minion' + ) raise SystemExit('\nExiting on Ctrl-c') From d20e5912758505f318e91cef9f39415907498648 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 13 Aug 2012 15:44:16 -0600 Subject: [PATCH 122/769] Add a verify_env option to the master and minion configs --- src/saltext/consul/__init__.py | 55 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index adace0f..d8c8501 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -31,17 +31,18 @@ def start(self): self.parse_args() try: - verify_env([ - self.config['pki_dir'], - os.path.join(self.config['pki_dir'], 'minions'), - os.path.join(self.config['pki_dir'], 'minions_pre'), - os.path.join(self.config['pki_dir'], 'minions_rejected'), - self.config['cachedir'], - os.path.join(self.config['cachedir'], 'jobs'), - os.path.dirname(self.config['log_file']), - self.config['sock_dir'], - ], self.config['user'], - permissive=self.config['permissive_pki_access']) + if self.config['verify_env']: + verify_env([ + self.config['pki_dir'], + os.path.join(self.config['pki_dir'], 'minions'), + os.path.join(self.config['pki_dir'], 'minions_pre'), + os.path.join(self.config['pki_dir'], 'minions_rejected'), + self.config['cachedir'], + os.path.join(self.config['cachedir'], 'jobs'), + os.path.dirname(self.config['log_file']), + self.config['sock_dir'], + ], self.config['user'], + permissive=self.config['permissive_pki_access']) except OSError, err: sys.exit(err.errno) @@ -76,14 +77,15 @@ def start(self): self.parse_args() try: - verify_env([ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - os.path.dirname(self.config['log_file']), - ], self.config['user'], - permissive=self.config['permissive_pki_access']) + if self.config['verify_env']: + verify_env([ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + os.path.dirname(self.config['log_file']), + ], self.config['user'], + permissive=self.config['permissive_pki_access']) except OSError, err: sys.exit(err.errno) @@ -117,13 +119,14 @@ def start(self): ''' self.parse_args() try: - verify_env([ - self.config['pki_dir'], self.config['cachedir'], - os.path.dirname(self.config['log_file']), - ], - self.config['user'], - permissive=self.config['permissive_pki_access'] - ) + if self.config['verify_env']: + verify_env([ + self.config['pki_dir'], self.config['cachedir'], + os.path.dirname(self.config['log_file']), + ], + self.config['user'], + permissive=self.config['permissive_pki_access'] + ) except OSError, err: sys.exit(err.errno) From eebef411f677a9de105b6c9572c413aad02bdcda Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 13 Aug 2012 16:28:41 -0600 Subject: [PATCH 123/769] don't merge ext_nodes with pillar lookup, fix #1816 --- src/saltext/consul/pillar/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 51b4193..144026c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -242,12 +242,6 @@ def top_matches(self, top): for item in data: if isinstance(item, string_types): matches[env].append(item) - ext_matches = self.client.ext_nodes() - for env in ext_matches: - if env in matches: - matches[env] = list(set(ext_matches[env]).union(matches[env])) - else: - matches[env] = ext_matches[env] return matches def render_pstate(self, sls, env, mods): From 24931a1d636294ded29e73eab10a20d4d3ff5b37 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 14 Aug 2012 23:01:25 -0600 Subject: [PATCH 124/769] Modify verify_env to only work on the pki_dir --- src/saltext/consul/__init__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d8c8501..ed3a06e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -41,8 +41,11 @@ def start(self): os.path.join(self.config['cachedir'], 'jobs'), os.path.dirname(self.config['log_file']), self.config['sock_dir'], - ], self.config['user'], - permissive=self.config['permissive_pki_access']) + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + self.config['pki_dir'], + ) except OSError, err: sys.exit(err.errno) @@ -84,8 +87,11 @@ def start(self): self.config['sock_dir'], self.config['extension_modules'], os.path.dirname(self.config['log_file']), - ], self.config['user'], - permissive=self.config['permissive_pki_access']) + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + self.config['pki_dir'], + ) except OSError, err: sys.exit(err.errno) @@ -125,7 +131,8 @@ def start(self): os.path.dirname(self.config['log_file']), ], self.config['user'], - permissive=self.config['permissive_pki_access'] + permissive=self.config['permissive_pki_access'], + self.config['pki_dir'], ) except OSError, err: sys.exit(err.errno) From 573658fb7665cb0f16baf5900eeaed36c493ef15 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 14 Aug 2012 23:04:56 -0600 Subject: [PATCH 125/769] fix arg passing in verify_env --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ed3a06e..6cc2b9b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -44,7 +44,7 @@ def start(self): ], self.config['user'], permissive=self.config['permissive_pki_access'], - self.config['pki_dir'], + pki_dir=self.config['pki_dir'], ) except OSError, err: sys.exit(err.errno) @@ -90,7 +90,7 @@ def start(self): ], self.config['user'], permissive=self.config['permissive_pki_access'], - self.config['pki_dir'], + pki_dir=self.config['pki_dir'], ) except OSError, err: sys.exit(err.errno) @@ -132,7 +132,7 @@ def start(self): ], self.config['user'], permissive=self.config['permissive_pki_access'], - self.config['pki_dir'], + pki_dir=self.config['pki_dir'], ) except OSError, err: sys.exit(err.errno) From 9b6d41afc68c417c68f03f9a62cacef6fba8cfa6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Aug 2012 15:38:57 -0600 Subject: [PATCH 126/769] Clean up unused imports and pylint bugs from Jenkins --- src/saltext/consul/__init__.py | 2 +- src/saltext/consul/pillar/__init__.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6cc2b9b..3879ffd 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,7 +7,7 @@ import sys import logging -# Import salt libs, the try block bypasses an issue at build time so that c +# Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail from salt.version import __version__ diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 144026c..8a40848 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -7,7 +7,6 @@ import copy import collections import logging -import subprocess # Import Salt libs import salt.loader @@ -17,10 +16,6 @@ from salt._compat import string_types from salt.template import compile_template -# Import third party libs -import zmq -import yaml - log = logging.getLogger(__name__) @@ -331,7 +326,7 @@ def ext_pillar(self): ext.update(self.ext_pillars[key](*val)) else: ext.update(self.ext_pillars[key](val)) - except Exception as e: + except Exception: log.critical('Failed to load ext_pillar {0}'.format(key)) return ext From c839d9ab9bd339934f05a40f88b2128c25e0212a Mon Sep 17 00:00:00 2001 From: Aaron Tygart Date: Thu, 13 Sep 2012 13:20:28 -0500 Subject: [PATCH 127/769] Clean up initial logging messages Minion logging was producing duplicate "Starting Minion" entries when subprocesses were called on Windows. Startup messages moved to bootstrappers. --- src/saltext/consul/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3879ffd..cf643d0 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -19,6 +19,7 @@ if e.args[0] != 'No module named _msgpack': raise + class Master(parsers.MasterOptionParser): ''' Creates a master server @@ -50,6 +51,7 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() + log = logging.getLogger('salt.master') # Late import so logging works correctly if not verify_socket(self.config['interface'], @@ -58,6 +60,7 @@ def start(self): self.exit(4, 'The ports are not available to bind') import salt.master + log.warn('Starting the Salt Master') master = salt.master.Master(self.config) self.daemonize_if_required() self.set_pidfile() @@ -96,6 +99,7 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() + log = logging.getLogger('salt.minion') # Late import so logging works correctly import salt.minion @@ -105,12 +109,13 @@ def start(self): # This is the latest safe place to daemonize self.daemonize_if_required() try: + log.warn('Starting the Salt Minion "{0}"'.format(self.config['id'])) minion = salt.minion.Minion(self.config) self.set_pidfile() if check_user(self.config['user']): minion.tune_in() except KeyboardInterrupt: - logging.getLogger(__name__).warn('Stopping the Salt Minion') + log.warn('Stopping the Salt Minion') raise SystemExit('\nExiting on Ctrl-c') @@ -137,8 +142,8 @@ def start(self): except OSError, err: sys.exit(err.errno) - self.setup_logfile_logger() + log = logging.getLogger('salt.syndic') # Late import so logging works correctly import salt.minion @@ -147,10 +152,10 @@ def start(self): if check_user(self.config['user']): try: + log.warn('Starting the Salt Syndic Minion "{0}"'.format( + self.config['id'])) syndic = salt.minion.Syndic(self.config) syndic.tune_in() except KeyboardInterrupt: - logging.getLogger(__name__).warn( - 'Stopping the Salt Syndic Minion' - ) + log.warn('Stopping the Salt Syndic Minion') raise SystemExit('\nExiting on Ctrl-c') From 3f8d47a7de8ef4d85b2b7be726a59fcf302a28f0 Mon Sep 17 00:00:00 2001 From: Sean Channel Date: Wed, 26 Sep 2012 11:45:26 -0700 Subject: [PATCH 128/769] missing line termination --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cf643d0..fe7ed66 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -57,7 +57,7 @@ def start(self): if not verify_socket(self.config['interface'], self.config['publish_port'], self.config['ret_port']): - self.exit(4, 'The ports are not available to bind') + self.exit(4, 'The ports are not available to bind\n') import salt.master log.warn('Starting the Salt Master') From d73d162b9c21d1ca3efe71379de38c959d456ced Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 30 Sep 2012 11:24:00 +0100 Subject: [PATCH 129/769] Improve/correct logging. * Change logging handler as per conversation on https://github.com/saltstack/salt/commit/25f9c5386ad09de6c9b7cdbf6f36e3c8ca1e2a0f#commitcomment-1889286 * Add some logging information to know who is the user starting the salt binaries. --- src/saltext/consul/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fe7ed66..4b412a7 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,6 +6,7 @@ import os import sys import logging +import getpass # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail @@ -51,7 +52,7 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - log = logging.getLogger('salt.master') + logging.getLogger(__name__).warn('Setting up the Salt Master') # Late import so logging works correctly if not verify_socket(self.config['interface'], @@ -60,7 +61,6 @@ def start(self): self.exit(4, 'The ports are not available to bind\n') import salt.master - log.warn('Starting the Salt Master') master = salt.master.Master(self.config) self.daemonize_if_required() self.set_pidfile() @@ -99,7 +99,11 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - log = logging.getLogger('salt.minion') + logging.getLogger(__name__).warn( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] + ) + ) # Late import so logging works correctly import salt.minion @@ -109,7 +113,6 @@ def start(self): # This is the latest safe place to daemonize self.daemonize_if_required() try: - log.warn('Starting the Salt Minion "{0}"'.format(self.config['id'])) minion = salt.minion.Minion(self.config) self.set_pidfile() if check_user(self.config['user']): @@ -143,7 +146,11 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - log = logging.getLogger('salt.syndic') + logging.getLogger(__name__).warn( + 'Setting up the Salt Syndic Minion "{0}"'.format( + self.config['id'] + ) + ) # Late import so logging works correctly import salt.minion @@ -152,8 +159,6 @@ def start(self): if check_user(self.config['user']): try: - log.warn('Starting the Salt Syndic Minion "{0}"'.format( - self.config['id'])) syndic = salt.minion.Syndic(self.config) syndic.tune_in() except KeyboardInterrupt: From b5b49aa81627dce2a59c77109c25818f7696d2b2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 1 Oct 2012 21:32:50 +0100 Subject: [PATCH 130/769] Fix `log` calls. --- src/saltext/consul/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4b412a7..90de7e9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -99,7 +99,8 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - logging.getLogger(__name__).warn( + log = logging.getLogger(__name__) + log.warn( 'Setting up the Salt Minion "{0}"'.format( self.config['id'] ) @@ -146,7 +147,8 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - logging.getLogger(__name__).warn( + log = logging.getLogger(__name__) + log.warn( 'Setting up the Salt Syndic Minion "{0}"'.format( self.config['id'] ) From ce964262fe9c186ae5ffcf6c80ea3550b934ec81 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 3 Oct 2012 22:15:47 +0100 Subject: [PATCH 131/769] Warn the user about grains not accurate in some situations and try to work around that. Fixes #2151. * Worked around the egg and the chicken problem regarding imports so that we can log on the grains module. * Added `salt.modules.cmdmod._run_all_quiet` which mimics `salt.modules.cmdmod._run_quiet` except it returns the whole command run info dict. * If `dmidecode` is not executed properly, default to `lspci` instead of producing bad results. --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 90de7e9..00681a6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,7 +13,6 @@ from salt.version import __version__ try: - import salt.config from salt.utils import parsers from salt.utils.verify import check_user, verify_env, verify_socket except ImportError as e: From 3952be6390e66a05035d60d0aee11c21cf32224f Mon Sep 17 00:00:00 2001 From: Aaron Tygart Date: Thu, 4 Oct 2012 10:57:09 -0500 Subject: [PATCH 132/769] Add stack trace to ext_pillar load failures. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8a40848..b09ae89 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -327,7 +327,7 @@ def ext_pillar(self): else: ext.update(self.ext_pillars[key](val)) except Exception: - log.critical('Failed to load ext_pillar {0}'.format(key)) + log.exception('Failed to load ext_pillar {0}'.format(key)) return ext def compile_pillar(self): From 6ea0c41cda25bc91454b8f01bd2e2ad8444d5aea Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 9 Oct 2012 14:04:21 -0600 Subject: [PATCH 133/769] Make authentication tokens work! --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 00681a6..9e51d7b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -42,6 +42,7 @@ def start(self): os.path.join(self.config['cachedir'], 'jobs'), os.path.dirname(self.config['log_file']), self.config['sock_dir'], + self.config['token_dir'], ], self.config['user'], permissive=self.config['permissive_pki_access'], From d95d77528aa169c640f429416bd035c2639d6f11 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 10 Oct 2012 23:53:01 -0600 Subject: [PATCH 134/769] Add pillar_opts option. This is turned on by default! This adds the master config file options to the pillar, this means that all of the config data on the master config is available via pillar now. Fix #1886 --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b09ae89..177b135 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -339,6 +339,8 @@ def compile_pillar(self): pillar, errors = self.render_pillar(matches) pillar.update(self.ext_pillar()) errors.extend(terrors) + if self.opts.get('pillar_opts', False): + pillar['master'] = self.opts if errors: for error in errors: log.critical('Pillar render error: {0}'.format(error)) From da628887603ac8083ee723beaa3e1433bf998c09 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 14 Oct 2012 15:43:23 -0600 Subject: [PATCH 135/769] Accept updated pillar system --- src/saltext/consul/pillar/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 177b135..9006714 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -52,11 +52,13 @@ def compile_pillar(self): load = {'id': self.id_, 'grains': self.grains, 'env': self.opts['environment'], + 'ver': '2', 'cmd': '_pillar'} - return self.auth.crypticle.loads( - self.sreq.send('aes', self.auth.crypticle.dumps(load), 3, 7200) - ) - + ret = self.sreq.send('aes', self.auth.crypticle.dumps(load), 3, 7200) + key = self.auth.get_keys() + aes = key.private_decrypt(ret['key'], 4) + pcrypt = salt.crypt.Crypticle(self.opts, aes) + return pcrypt.loads(ret['pillar']) class Pillar(object): From 56237ca5236a97664cd2ad130454c72590558eaf Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 17 Oct 2012 19:16:04 -0600 Subject: [PATCH 136/769] Fix #2240 --- src/saltext/consul/pillar/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 9006714..388e89c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -70,15 +70,18 @@ def __init__(self, opts, grains, id_, env): self.opts = self.__gen_opts(opts, grains, id_, env) self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) - self.functions = salt.loader.minion_mods(self.opts) + if opts['file_client'] == 'local': + self.functions = salt.loader.minion_mods(opts) + else: + self.functions = salt.loader.minion_mods(self.opts) self.rend = salt.loader.render(self.opts, self.functions) self.ext_pillars = salt.loader.pillars(self.opts, self.functions) - def __gen_opts(self, opts, grains, id_, env=None): + def __gen_opts(self, opts_in, grains, id_, env=None): ''' The options need to be altered to conform to the file client ''' - opts = copy.deepcopy(opts) + opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' opts['grains'] = grains From 98dac1ac45a5e86cc3e7ad5d3a413bd0a7d34a7f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 17 Oct 2012 22:55:35 -0600 Subject: [PATCH 137/769] Fix issue with file_client not being in the master opts --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 388e89c..3ca02a3 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -70,7 +70,7 @@ def __init__(self, opts, grains, id_, env): self.opts = self.__gen_opts(opts, grains, id_, env) self.client = salt.fileclient.get_file_client(self.opts) self.matcher = salt.minion.Matcher(self.opts) - if opts['file_client'] == 'local': + if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) else: self.functions = salt.loader.minion_mods(self.opts) From f875bd7082d19ae65aab3216427f0c81e8462048 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 22 Oct 2012 15:14:21 -0600 Subject: [PATCH 138/769] Fix memory mapping issue --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3ca02a3..24ca331 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -69,11 +69,11 @@ def __init__(self, opts, grains, id_, env): # use the local file client self.opts = self.__gen_opts(opts, grains, id_, env) self.client = salt.fileclient.get_file_client(self.opts) - self.matcher = salt.minion.Matcher(self.opts) if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) else: self.functions = salt.loader.minion_mods(self.opts) + self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) self.ext_pillars = salt.loader.pillars(self.opts, self.functions) From b068e3abe19595129b3f44084b9313f23762cc8a Mon Sep 17 00:00:00 2001 From: Jakub Vysoky Date: Fri, 26 Oct 2012 14:35:21 +0200 Subject: [PATCH 139/769] pass pillar to ext_pillar so it can manipulate it --- src/saltext/consul/pillar/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 24ca331..ba2484f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -304,7 +304,7 @@ def render_pillar(self, matches): errors += err return pillar, errors - def ext_pillar(self): + def ext_pillar(self, pillar={}): ''' Render the external pillar data ''' @@ -326,11 +326,11 @@ def ext_pillar(self): continue try: if isinstance(val, dict): - ext.update(self.ext_pillars[key](**val)) + ext.update(self.ext_pillars[key](pillar=pillar, **val)) elif isinstance(val, list): - ext.update(self.ext_pillars[key](*val)) + ext.update(self.ext_pillars[key](*val, pillar=pillar)) else: - ext.update(self.ext_pillars[key](val)) + ext.update(self.ext_pillars[key](val, pillar=pillar)) except Exception: log.exception('Failed to load ext_pillar {0}'.format(key)) return ext @@ -342,7 +342,7 @@ def compile_pillar(self): top, terrors = self.get_top() matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar.update(self.ext_pillar()) + pillar.update(self.ext_pillar(pillar=pillar)) errors.extend(terrors) if self.opts.get('pillar_opts', False): pillar['master'] = self.opts From 3632815b7e122431e9cc03e87ec6813faf202a24 Mon Sep 17 00:00:00 2001 From: Jakub Vysoky Date: Fri, 26 Oct 2012 17:41:18 +0200 Subject: [PATCH 140/769] update pillars directly and omit default args default args can be painful --- src/saltext/consul/pillar/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ba2484f..18f8e14 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -304,7 +304,7 @@ def render_pillar(self, matches): errors += err return pillar, errors - def ext_pillar(self, pillar={}): + def ext_pillar(self, pillar): ''' Render the external pillar data ''' @@ -313,7 +313,6 @@ def ext_pillar(self, pillar={}): if not isinstance(self.opts['ext_pillar'], list): log.critical('The "ext_pillar" option is malformed') return {} - ext = {} for run in self.opts['ext_pillar']: if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') @@ -326,14 +325,14 @@ def ext_pillar(self, pillar={}): continue try: if isinstance(val, dict): - ext.update(self.ext_pillars[key](pillar=pillar, **val)) + pillar.update(self.ext_pillars[key](pillar, **val)) elif isinstance(val, list): - ext.update(self.ext_pillars[key](*val, pillar=pillar)) + pillar.update(self.ext_pillars[key](pillar, *val)) else: - ext.update(self.ext_pillars[key](val, pillar=pillar)) + pillar.update(self.ext_pillars[key](pillar, val)) except Exception: log.exception('Failed to load ext_pillar {0}'.format(key)) - return ext + return pillar def compile_pillar(self): ''' @@ -342,7 +341,7 @@ def compile_pillar(self): top, terrors = self.get_top() matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar.update(self.ext_pillar(pillar=pillar)) + self.ext_pillar(pillar) errors.extend(terrors) if self.opts.get('pillar_opts', False): pillar['master'] = self.opts From e22ed27c83c418ce95767fa0e7986d74cc083a1c Mon Sep 17 00:00:00 2001 From: Jakub Vysoky Date: Mon, 29 Oct 2012 16:00:44 +0100 Subject: [PATCH 141/769] do a recursive update of rendered and ext pillars --- src/saltext/consul/pillar/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 18f8e14..6b08508 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -4,7 +4,6 @@ # Import python libs import os -import copy import collections import logging @@ -15,6 +14,7 @@ import salt.crypt from salt._compat import string_types from salt.template import compile_template +from salt.utils.dictupdate import update log = logging.getLogger(__name__) @@ -325,11 +325,12 @@ def ext_pillar(self, pillar): continue try: if isinstance(val, dict): - pillar.update(self.ext_pillars[key](pillar, **val)) + ext = self.ext_pillars[key](pillar, **val) elif isinstance(val, list): - pillar.update(self.ext_pillars[key](pillar, *val)) + ext = self.ext_pillars[key](pillar, *val) else: - pillar.update(self.ext_pillars[key](pillar, val)) + ext = self.ext_pillars[key](pillar, val) + update(pillar, ext) except Exception: log.exception('Failed to load ext_pillar {0}'.format(key)) return pillar From 44a42f429a5230dcc19abb792c8791ed9a2ac8f2 Mon Sep 17 00:00:00 2001 From: Jakub Vysoky Date: Mon, 29 Oct 2012 16:02:20 +0100 Subject: [PATCH 142/769] always better to log whole exception --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6b08508..55a0a01 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -331,8 +331,8 @@ def ext_pillar(self, pillar): else: ext = self.ext_pillars[key](pillar, val) update(pillar, ext) - except Exception: - log.exception('Failed to load ext_pillar {0}'.format(key)) + except Exception as exc: + log.exception('Failed to load ext_pillar {0}: {1}'.format(key, exc)) return pillar def compile_pillar(self): From 88e9cf2aafad53d276ee97593364dda02189e11f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 31 Oct 2012 14:07:14 -0600 Subject: [PATCH 143/769] Don't pass through the minions grains back in the pillar --- src/saltext/consul/pillar/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 55a0a01..96ca88d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -344,8 +344,13 @@ def compile_pillar(self): pillar, errors = self.render_pillar(matches) self.ext_pillar(pillar) errors.extend(terrors) - if self.opts.get('pillar_opts', False): - pillar['master'] = self.opts + if self.opts.get('pillar_opts', True): + mopts = dict(self.opts) + if 'grains' in mopts: + mopts.pop('grains') + if 'aes' in mopts: + mopts.pop('aes') + pillar['master'] = mopts if errors: for error in errors: log.critical('Pillar render error: {0}'.format(error)) From ef5227e69019a2d5d1732153d46c948a74767697 Mon Sep 17 00:00:00 2001 From: Mike Chesnut Date: Thu, 1 Nov 2012 18:40:19 -0700 Subject: [PATCH 144/769] add proc subdir of cachedir to the verify_env dir list this fixes an issue for me when I'm running minions as non-root users; without it I get a permission denied error when the minion starts up --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9e51d7b..0a7e6da 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -40,6 +40,7 @@ def start(self): os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], os.path.join(self.config['cachedir'], 'jobs'), + os.path.join(self.config['cachedir'], 'proc'), os.path.dirname(self.config['log_file']), self.config['sock_dir'], self.config['token_dir'], From 70254e914ade55ae02cd390aa81a8265ce93396e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 4 Nov 2012 17:47:31 -0700 Subject: [PATCH 145/769] Another commit error --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0a7e6da..8185208 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -49,7 +49,7 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - except OSError, err: + except OSError as err: sys.exit(err.errno) self.setup_logfile_logger() From 80746d10016e74511f8549989201eca1e32eede3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 4 Nov 2012 17:53:02 -0700 Subject: [PATCH 146/769] Fix more exception syntax --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8185208..80b82de 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -96,7 +96,7 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - except OSError, err: + except OSError as err: sys.exit(err.errno) self.setup_logfile_logger() @@ -144,7 +144,7 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - except OSError, err: + except OSError as err: sys.exit(err.errno) self.setup_logfile_logger() From a907d5db5d99ea725b6591a9c10c4324fe6d544b Mon Sep 17 00:00:00 2001 From: David Boucha Date: Fri, 9 Nov 2012 15:48:34 -0700 Subject: [PATCH 147/769] Add option to have minion retry dns If name resolution fails when the minion starts, allow the minion to retry dns resolution after a configurable number of seconds. Defaults to 30 seconds. Add retry_dns option to minion config. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 80b82de..e683fd4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -81,6 +81,7 @@ def start(self): ''' Execute this method to start up a minion. ''' + self.daemonize_if_required() self.parse_args() try: @@ -113,7 +114,6 @@ def start(self): # waiting for it, if we daemonize later then the minion could halt # the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize - self.daemonize_if_required() try: minion = salt.minion.Minion(self.config) self.set_pidfile() From 0fd62eb42f454ba9830e0d9db8095844d174a32e Mon Sep 17 00:00:00 2001 From: saltstack Date: Fri, 9 Nov 2012 20:55:59 -0700 Subject: [PATCH 148/769] Move the dns retry loop to config.py --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e683fd4..80b82de 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -81,7 +81,6 @@ def start(self): ''' Execute this method to start up a minion. ''' - self.daemonize_if_required() self.parse_args() try: @@ -114,6 +113,7 @@ def start(self): # waiting for it, if we daemonize later then the minion could halt # the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize + self.daemonize_if_required() try: minion = salt.minion.Minion(self.config) self.set_pidfile() From 477129f814f3256efe3b851a1c0de5079437d11d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 12 Nov 2012 21:21:12 -0700 Subject: [PATCH 149/769] remove unused getpass --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 80b82de..626451d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,7 +6,6 @@ import os import sys import logging -import getpass # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail From 8b1e082181b4641657999eca346bf64c07074b01 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 15 Nov 2012 16:20:24 -0700 Subject: [PATCH 150/769] Fix for #2567 --- src/saltext/consul/pillar/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 96ca88d..bd09f25 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -70,6 +70,7 @@ def __init__(self, opts, grains, id_, env): self.opts = self.__gen_opts(opts, grains, id_, env) self.client = salt.fileclient.get_file_client(self.opts) if opts.get('file_client', '') == 'local': + opts['grains'] = grains self.functions = salt.loader.minion_mods(opts) else: self.functions = salt.loader.minion_mods(self.opts) From b3c725c5445c1690b132b5f4f206473e0a7b5c2c Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 19 Nov 2012 15:18:08 -0700 Subject: [PATCH 151/769] Import migration util. Test it now --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 626451d..7c825f2 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,6 +10,7 @@ # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail from salt.version import __version__ +from salt.utils.migrations import migrate_paths try: from salt.utils import parsers @@ -29,6 +30,7 @@ def start(self): Run the sequence to start a salt master server ''' self.parse_args() + migrate_paths() try: if self.config['verify_env']: From 136182a8ecd1d2f74237562aab565b79aeb225dc Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 19 Nov 2012 15:38:50 -0700 Subject: [PATCH 152/769] add master option to path migration --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7c825f2..0051a12 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -30,7 +30,7 @@ def start(self): Run the sequence to start a salt master server ''' self.parse_args() - migrate_paths() + migrate_paths('master') try: if self.config['verify_env']: From deeb62e3e79a42924b41273fe143bb45341ee4de Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 19 Nov 2012 15:41:37 -0700 Subject: [PATCH 153/769] move migrate_paths before parse_args --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0051a12..7c5e0a4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -29,8 +29,8 @@ def start(self): ''' Run the sequence to start a salt master server ''' - self.parse_args() migrate_paths('master') + self.parse_args() try: if self.config['verify_env']: From 3378e3488278e72952d63d2241812324cd9a0dfe Mon Sep 17 00:00:00 2001 From: David Boucha Date: Tue, 20 Nov 2012 12:48:49 -0700 Subject: [PATCH 154/769] Get migration to work. --- src/saltext/consul/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7c5e0a4..f048683 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,7 +10,6 @@ # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail from salt.version import __version__ -from salt.utils.migrations import migrate_paths try: from salt.utils import parsers @@ -29,7 +28,6 @@ def start(self): ''' Run the sequence to start a salt master server ''' - migrate_paths('master') self.parse_args() try: @@ -103,9 +101,7 @@ def start(self): self.setup_logfile_logger() log = logging.getLogger(__name__) log.warn( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] - ) + 'Setting up the Salt Minion "{0}"'.format( self.config['id']) ) # Late import so logging works correctly From af5f384b7c33f9dcc183e3c12134ccf414ae6e33 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 28 Nov 2012 16:53:57 -0700 Subject: [PATCH 155/769] Try changing location of the pki migrations --- src/saltext/consul/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f048683..55ef945 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,6 +10,7 @@ # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail from salt.version import __version__ +import salt.utils.migrations try: from salt.utils import parsers @@ -59,7 +60,7 @@ def start(self): self.config['publish_port'], self.config['ret_port']): self.exit(4, 'The ports are not available to bind\n') - + salt.utils.migrations.migrate_paths(self.config) import salt.master master = salt.master.Master(self.config) self.daemonize_if_required() @@ -103,7 +104,7 @@ def start(self): log.warn( 'Setting up the Salt Minion "{0}"'.format( self.config['id']) ) - + salt.utils.migrations.migrate_paths(self.config) # Late import so logging works correctly import salt.minion # If the minion key has not been accepted, then Salt enters a loop From 3f33d89a1aa9cbc5de22039487bbb3c2b7b731fa Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 28 Nov 2012 17:07:59 -0700 Subject: [PATCH 156/769] move migration to before dropping privileges --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 55ef945..348581f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -10,7 +10,7 @@ # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail from salt.version import __version__ -import salt.utils.migrations +from salt.utils import migrations try: from salt.utils import parsers @@ -60,7 +60,7 @@ def start(self): self.config['publish_port'], self.config['ret_port']): self.exit(4, 'The ports are not available to bind\n') - salt.utils.migrations.migrate_paths(self.config) + migrations.migrate_paths(self.config) import salt.master master = salt.master.Master(self.config) self.daemonize_if_required() @@ -104,7 +104,7 @@ def start(self): log.warn( 'Setting up the Salt Minion "{0}"'.format( self.config['id']) ) - salt.utils.migrations.migrate_paths(self.config) + migrations.migrate_paths(self.config) # Late import so logging works correctly import salt.minion # If the minion key has not been accepted, then Salt enters a loop From 44535e099b7e8a8e62c95cc7d155d24cf79da735 Mon Sep 17 00:00:00 2001 From: Henrik Holmboe Date: Thu, 6 Dec 2012 23:46:21 +0100 Subject: [PATCH 157/769] Comments and whitespace in order to harmonize them across the source code. Mostly changes in and around the imports. A few imports were moved around but there should be no effect on the code (hopefully). --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bd09f25..347aa9b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -7,7 +7,7 @@ import collections import logging -# Import Salt libs +# Import salt libs import salt.loader import salt.fileclient import salt.minion From 47cfafe390c1196107878cf1ca89398f08e2e51f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 12 Dec 2012 12:47:45 -0700 Subject: [PATCH 158/769] The syndic needs to verify more directories --- src/saltext/consul/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 348581f..128523a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -135,12 +135,15 @@ def start(self): try: if self.config['verify_env']: verify_env([ - self.config['pki_dir'], self.config['cachedir'], - os.path.dirname(self.config['log_file']), - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + os.path.dirname(self.config['log_file']), + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) except OSError as err: sys.exit(err.errno) From 1f47b947968c425d174264faaced2c96fba8dbfb Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 7 Jan 2013 23:22:23 -0700 Subject: [PATCH 159/769] Move logfile verification into the new verify_files function Fix #3156 --- src/saltext/consul/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 128523a..2f3e014 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -15,6 +15,7 @@ try: from salt.utils import parsers from salt.utils.verify import check_user, verify_env, verify_socket + from salt.utils.verify import verify_files except ImportError as e: if e.args[0] != 'No module named _msgpack': raise @@ -41,7 +42,6 @@ def start(self): self.config['cachedir'], os.path.join(self.config['cachedir'], 'jobs'), os.path.join(self.config['cachedir'], 'proc'), - os.path.dirname(self.config['log_file']), self.config['sock_dir'], self.config['token_dir'], ], @@ -49,6 +49,9 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) + verify_files( + [self.config['log_file']], + self.config['user']) except OSError as err: sys.exit(err.errno) @@ -90,12 +93,14 @@ def start(self): self.config['cachedir'], self.config['sock_dir'], self.config['extension_modules'], - os.path.dirname(self.config['log_file']), ], self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) + verify_files( + [self.config['log_file']], + self.config['user']) except OSError as err: sys.exit(err.errno) @@ -139,12 +144,14 @@ def start(self): self.config['cachedir'], self.config['sock_dir'], self.config['extension_modules'], - os.path.dirname(self.config['log_file']), ], self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) + verify_files( + [self.config['log_file']], + self.config['user']) except OSError as err: sys.exit(err.errno) From dcdaef969cd32ba42f3f89dc163ad0207b7bf8e5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 8 Jan 2013 12:11:12 +0000 Subject: [PATCH 160/769] Only verify log file in not using syslog. Refs #3156. --- src/saltext/consul/__init__.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 2f3e014..ac4d38b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -49,21 +49,27 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - verify_files( - [self.config['log_file']], - self.config['user']) + if (not self.config['log_file'].startswith('tcp://') or + not self.config['log_file'].startswith('udp://') or + not self.config['log_file'].startswith('file://')): + # Logfile is not using Syslog, verify + verify_files( + [self.config['log_file']], + self.config['user'] + ) except OSError as err: sys.exit(err.errno) self.setup_logfile_logger() logging.getLogger(__name__).warn('Setting up the Salt Master') - # Late import so logging works correctly if not verify_socket(self.config['interface'], self.config['publish_port'], self.config['ret_port']): self.exit(4, 'The ports are not available to bind\n') migrations.migrate_paths(self.config) + + # Late import so logging works correctly import salt.master master = salt.master.Master(self.config) self.daemonize_if_required() @@ -98,6 +104,14 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) + if (not self.config['log_file'].startswith('tcp://') or + not self.config['log_file'].startswith('udp://') or + not self.config['log_file'].startswith('file://')): + # Logfile is not using Syslog, verify + verify_files( + [self.config['log_file']], + self.config['user'] + ) verify_files( [self.config['log_file']], self.config['user']) @@ -149,9 +163,14 @@ def start(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - verify_files( - [self.config['log_file']], - self.config['user']) + if (not self.config['log_file'].startswith('tcp://') or + not self.config['log_file'].startswith('udp://') or + not self.config['log_file'].startswith('file://')): + # Logfile is not using Syslog, verify + verify_files( + [self.config['log_file']], + self.config['user'] + ) except OSError as err: sys.exit(err.errno) From 0535d7a9431bbc4ed4209b87e62d98aff4bcebc7 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 11 Jan 2013 12:51:39 -0700 Subject: [PATCH 161/769] Fix saltstack/salt-bootstrap#10 --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ac4d38b..3fc39b1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -94,11 +94,15 @@ def start(self): try: if self.config['verify_env']: + confd = os.path.join( + os.path.dirname(self.config['conf_file']), + 'minion.d') verify_env([ self.config['pki_dir'], self.config['cachedir'], self.config['sock_dir'], self.config['extension_modules'], + confd, ], self.config['user'], permissive=self.config['permissive_pki_access'], From 520c8103d1137bc55ba2d7eefedd4b4b097324c0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 13 Jan 2013 15:28:23 +0000 Subject: [PATCH 162/769] Simplify sub-classing `Master`, `Minion` and `Syndic` + PEP-8. * Since we no longer suffer from too early logger initiation, replace any `logging.getLogger(__name__)` calls with an initial defined `logger`(not `log` so it does not clash with `salt.log`. --- src/saltext/consul/__init__.py | 161 ++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 51 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3fc39b1..c198144 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -9,7 +9,7 @@ # Import salt libs, the try block bypasses an issue at build time so that # modules don't cause the build to fail -from salt.version import __version__ +from salt.version import __version__ # pylint: disable-msg=W402 from salt.utils import migrations try: @@ -21,12 +21,15 @@ raise +logger = logging.getLogger(__name__) + + class Master(parsers.MasterOptionParser): ''' Creates a master server ''' - def start(self): + def run(self): ''' Run the sequence to start a salt master server ''' @@ -34,20 +37,22 @@ def start(self): try: if self.config['verify_env']: - verify_env([ - self.config['pki_dir'], - os.path.join(self.config['pki_dir'], 'minions'), - os.path.join(self.config['pki_dir'], 'minions_pre'), - os.path.join(self.config['pki_dir'], 'minions_rejected'), - self.config['cachedir'], - os.path.join(self.config['cachedir'], 'jobs'), - os.path.join(self.config['cachedir'], 'proc'), - self.config['sock_dir'], - self.config['token_dir'], - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + verify_env( + [ + self.config['pki_dir'], + os.path.join(self.config['pki_dir'], 'minions'), + os.path.join(self.config['pki_dir'], 'minions_pre'), + os.path.join(self.config['pki_dir'], + 'minions_rejected'), + self.config['cachedir'], + os.path.join(self.config['cachedir'], 'jobs'), + os.path.join(self.config['cachedir'], 'proc'), + self.config['sock_dir'], + self.config['token_dir'], + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) if (not self.config['log_file'].startswith('tcp://') or not self.config['log_file'].startswith('udp://') or @@ -61,7 +66,7 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - logging.getLogger(__name__).warn('Setting up the Salt Master') + logger.warn('Setting up the Salt Master') if not verify_socket(self.config['interface'], self.config['publish_port'], @@ -71,22 +76,39 @@ def start(self): # Late import so logging works correctly import salt.master - master = salt.master.Master(self.config) + self.master = salt.master.Master(self.config) self.daemonize_if_required() self.set_pidfile() if check_user(self.config['user']): try: - master.start() + self.start() except salt.master.MasterExit: + self.stop() + finally: sys.exit() + def start(self): + ''' + Start the actual master. If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).start() + + NOTE: Run your start-up code before calling `super()`. + ''' + self.master.start() + + def stop(self): + ''' + If sub-classed, run any shutdown operations on this method. + ''' + class Minion(parsers.MinionOptionParser): ''' Create a minion server ''' - def start(self): + def run(self): ''' Execute this method to start up a minion. ''' @@ -95,18 +117,20 @@ def start(self): try: if self.config['verify_env']: confd = os.path.join( - os.path.dirname(self.config['conf_file']), - 'minion.d') - verify_env([ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + os.path.dirname(self.config['conf_file']), + 'minion.d' + ) + verify_env( + [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) if (not self.config['log_file'].startswith('tcp://') or not self.config['log_file'].startswith('udp://') or @@ -123,9 +147,10 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - log = logging.getLogger(__name__) - log.warn( - 'Setting up the Salt Minion "{0}"'.format( self.config['id']) + logger.warn( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] + ) ) migrations.migrate_paths(self.config) # Late import so logging works correctly @@ -136,36 +161,54 @@ def start(self): # This is the latest safe place to daemonize self.daemonize_if_required() try: - minion = salt.minion.Minion(self.config) + self.minion = salt.minion.Minion(self.config) self.set_pidfile() if check_user(self.config['user']): - minion.tune_in() + self.start() except KeyboardInterrupt: log.warn('Stopping the Salt Minion') + self.stop() + finally: raise SystemExit('\nExiting on Ctrl-c') + def start(self): + ''' + Start the actual minion. If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).start() + + NOTE: Run your start-up code before calling `super()`. + ''' + self.minion.tune_in() + + def stop(self): + ''' + If sub-classed, run any shutdown operations on this method. + ''' + class Syndic(parsers.SyndicOptionParser): ''' Create a syndic server ''' - def start(self): + def run(self): ''' Execute this method to start up a syndic. ''' self.parse_args() try: if self.config['verify_env']: - verify_env([ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + verify_env( + [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) if (not self.config['log_file'].startswith('tcp://') or not self.config['log_file'].startswith('udp://') or @@ -179,8 +222,7 @@ def start(self): sys.exit(err.errno) self.setup_logfile_logger() - log = logging.getLogger(__name__) - log.warn( + logger.warn( 'Setting up the Salt Syndic Minion "{0}"'.format( self.config['id'] ) @@ -193,8 +235,25 @@ def start(self): if check_user(self.config['user']): try: - syndic = salt.minion.Syndic(self.config) - syndic.tune_in() + self.syndic = salt.minion.Syndic(self.config) + self.start() except KeyboardInterrupt: log.warn('Stopping the Salt Syndic Minion') + self.stop() + finally: raise SystemExit('\nExiting on Ctrl-c') + + def start(self): + ''' + Start the actual syndic. If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).start() + + NOTE: Run your start-up code before calling `super()`. + ''' + self.syndic.tune_in() + + def stop(self): + ''' + If sub-classed, run any shutdown operations on this method. + ''' From 8e243875c617fa5b7fc204956a950526c7eef2f7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 14 Jan 2013 23:15:48 +0000 Subject: [PATCH 163/769] Comply with the convention and still provide the ability to subclass masters and minions. --- src/saltext/consul/__init__.py | 111 +++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 47 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c198144..4aa68b9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,8 +7,8 @@ import sys import logging -# Import salt libs, the try block bypasses an issue at build time so that -# modules don't cause the build to fail +# Import salt libs, the try block below bypasses an issue at build time so +# that modules don't cause the build to fail from salt.version import __version__ # pylint: disable-msg=W402 from salt.utils import migrations @@ -29,9 +29,13 @@ class Master(parsers.MasterOptionParser): Creates a master server ''' - def run(self): + def prepare(self): ''' - Run the sequence to start a salt master server + Run the preparation sequence required to start a salt master server. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).prepare() ''' self.parse_args() @@ -79,25 +83,27 @@ def run(self): self.master = salt.master.Master(self.config) self.daemonize_if_required() self.set_pidfile() - if check_user(self.config['user']): - try: - self.start() - except salt.master.MasterExit: - self.stop() - finally: - sys.exit() def start(self): ''' - Start the actual master. If sub-classed, don't **ever** forget to run: + Start the actual master. + + If sub-classed, don't **ever** forget to run: super(YourSubClass, self).start() - NOTE: Run your start-up code before calling `super()`. + NOTE: Run any required code before calling `super()`. ''' - self.master.start() + self.prepare() + if check_user(self.config['user']): + try: + self.master.start() + except salt.master.MasterExit: + self.shutdown() + finally: + sys.exit() - def stop(self): + def shutdown(self): ''' If sub-classed, run any shutdown operations on this method. ''' @@ -108,9 +114,13 @@ class Minion(parsers.MinionOptionParser): Create a minion server ''' - def run(self): + def prepare(self): ''' - Execute this method to start up a minion. + Run the preparation sequence required to start a salt minion. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).prepare() ''' self.parse_args() @@ -160,28 +170,30 @@ def run(self): # the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize self.daemonize_if_required() - try: - self.minion = salt.minion.Minion(self.config) - self.set_pidfile() - if check_user(self.config['user']): - self.start() - except KeyboardInterrupt: - log.warn('Stopping the Salt Minion') - self.stop() - finally: - raise SystemExit('\nExiting on Ctrl-c') + self.minion = salt.minion.Minion(self.config) + self.set_pidfile() def start(self): ''' - Start the actual minion. If sub-classed, don't **ever** forget to run: + Start the actual minion. + + If sub-classed, don't **ever** forget to run: super(YourSubClass, self).start() - NOTE: Run your start-up code before calling `super()`. + NOTE: Run any required code before calling `super()`. ''' - self.minion.tune_in() + self.prepare() + try: + if check_user(self.config['user']): + self.minion.tune_in() + except KeyboardInterrupt: + logger.warn('Stopping the Salt Minion') + self.shutdown() + finally: + raise SystemExit('\nExiting on Ctrl-c') - def stop(self): + def shutdown(self): ''' If sub-classed, run any shutdown operations on this method. ''' @@ -192,9 +204,13 @@ class Syndic(parsers.SyndicOptionParser): Create a syndic server ''' - def run(self): + def prepare(self): ''' - Execute this method to start up a syndic. + Run the preparation sequence required to start a salt syndic minion. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).prepare() ''' self.parse_args() try: @@ -231,29 +247,30 @@ def run(self): # Late import so logging works correctly import salt.minion self.daemonize_if_required() + self.syndic = salt.minion.Syndic(self.config) self.set_pidfile() - if check_user(self.config['user']): - try: - self.syndic = salt.minion.Syndic(self.config) - self.start() - except KeyboardInterrupt: - log.warn('Stopping the Salt Syndic Minion') - self.stop() - finally: - raise SystemExit('\nExiting on Ctrl-c') - def start(self): ''' - Start the actual syndic. If sub-classed, don't **ever** forget to run: + Start the actual syndic. + + If sub-classed, don't **ever** forget to run: super(YourSubClass, self).start() - NOTE: Run your start-up code before calling `super()`. + NOTE: Run any required code before calling `super()`. ''' - self.syndic.tune_in() + self.prepare() + if check_user(self.config['user']): + try: + self.syndic.tune_in() + except KeyboardInterrupt: + logger.warn('Stopping the Salt Syndic Minion') + self.shutdown() + finally: + raise SystemExit('\nExiting on Ctrl-c') - def stop(self): + def shutdown(self): ''' If sub-classed, run any shutdown operations on this method. ''' From e9a04cbff3c77c2cec3d8f19ac27a0be13caf9eb Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 20 Jan 2013 03:16:07 +0000 Subject: [PATCH 164/769] Protect against the `log_path` being set to `None`. --- src/saltext/consul/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4aa68b9..c7e0009 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -58,14 +58,16 @@ def prepare(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - if (not self.config['log_file'].startswith('tcp://') or - not self.config['log_file'].startswith('udp://') or - not self.config['log_file'].startswith('file://')): + logfile = self.config['log_file'] + if logfile is not None and ( + not logfile.startswith('tcp://') or + not logfile.startswith('udp://') or + not logfile.startswith('file://')): # Logfile is not using Syslog, verify - verify_files( - [self.config['log_file']], - self.config['user'] - ) + verify_files( + [logfile], + self.config['user'] + ) except OSError as err: sys.exit(err.errno) From c8e5d0b7181da459179a10823f010a9aad652309 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 24 Jan 2013 21:58:55 -0700 Subject: [PATCH 165/769] A bunch of finally statements are killing legitimate error messages Fix #3412 or rather let the real issue float up --- src/saltext/consul/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c7e0009..d13f052 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -192,8 +192,6 @@ def start(self): except KeyboardInterrupt: logger.warn('Stopping the Salt Minion') self.shutdown() - finally: - raise SystemExit('\nExiting on Ctrl-c') def shutdown(self): ''' @@ -269,8 +267,6 @@ def start(self): except KeyboardInterrupt: logger.warn('Stopping the Salt Syndic Minion') self.shutdown() - finally: - raise SystemExit('\nExiting on Ctrl-c') def shutdown(self): ''' From b3b404afa0bee71e3a06d17f293bf383f37f900d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 28 Jan 2013 16:56:28 -0700 Subject: [PATCH 166/769] clean up some newlines --- src/saltext/consul/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d13f052..d4d6d25 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -28,7 +28,6 @@ class Master(parsers.MasterOptionParser): ''' Creates a master server ''' - def prepare(self): ''' Run the preparation sequence required to start a salt master server. @@ -115,7 +114,6 @@ class Minion(parsers.MinionOptionParser): ''' Create a minion server ''' - def prepare(self): ''' Run the preparation sequence required to start a salt minion. From 5423fa45a3c4fb0cf9d6712c9bdf62606c9dacbd Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Tue, 29 Jan 2013 21:49:03 -0500 Subject: [PATCH 167/769] Add support for #3483([pydsl] Import Python definitons from included files). --- src/saltext/consul/pillar/__init__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 347aa9b..53eb353 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -18,18 +18,15 @@ log = logging.getLogger(__name__) - +PILLAR = None def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option ''' - try: - return { - 'remote': RemotePillar, - 'local': Pillar - }.get(opts['file_client'], 'local')(opts, grains, id_, env) - except KeyError: - return Pillar(opts, grains, id_, env) + return { + 'remote': RemotePillar, + 'local': Pillar + }.get(opts['file_client'], Pillar)(opts, grains, id_, env) class RemotePillar(object): From 5487e5e28008a8f4b108e9bf9350ac6f93c1a4c7 Mon Sep 17 00:00:00 2001 From: Jack Kuan Date: Mon, 4 Feb 2013 22:07:41 -0500 Subject: [PATCH 168/769] Make getting the fileclient a bit more robust and remove unused PILLAR variable. --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 53eb353..67480e5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -18,7 +18,6 @@ log = logging.getLogger(__name__) -PILLAR = None def get_pillar(opts, grains, id_, env=None): ''' Return the correct pillar driver based on the file_client option From f9677524a58f7b6d47c8a3d330b8796bd4ead3a5 Mon Sep 17 00:00:00 2001 From: Cerales Date: Tue, 5 Feb 2013 20:37:20 +1100 Subject: [PATCH 169/769] Add a useful error message when the salt minion must exit because the IPC URI is too long. Using isinstance() is a bit messay, but I didn't want to step on the toes of handling the KeyboardInterrupt. --- src/saltext/consul/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4aa68b9..3e05e58 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -19,7 +19,7 @@ except ImportError as e: if e.args[0] != 'No module named _msgpack': raise - +from salt.exceptions import SaltSystemExit logger = logging.getLogger(__name__) @@ -187,11 +187,14 @@ def start(self): try: if check_user(self.config['user']): self.minion.tune_in() - except KeyboardInterrupt: + except (KeyboardInterrupt, SaltSystemExit) as e: logger.warn('Stopping the Salt Minion') - self.shutdown() + if isinstance(e, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(e)) finally: - raise SystemExit('\nExiting on Ctrl-c') + self.shutdown() def shutdown(self): ''' From fe7940a7909597eb1d83b122fb1f2c065a6d8394 Mon Sep 17 00:00:00 2001 From: Cerales Date: Tue, 5 Feb 2013 20:37:20 +1100 Subject: [PATCH 170/769] Add a useful error message when the salt minion must exit because the IPC URI is too long. Using isinstance() is a bit messay, but I didn't want to step on the toes of handling the KeyboardInterrupt. --- src/saltext/consul/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d4d6d25..6c79390 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -19,7 +19,7 @@ except ImportError as e: if e.args[0] != 'No module named _msgpack': raise - +from salt.exceptions import SaltSystemExit logger = logging.getLogger(__name__) @@ -187,8 +187,13 @@ def start(self): try: if check_user(self.config['user']): self.minion.tune_in() - except KeyboardInterrupt: + except (KeyboardInterrupt, SaltSystemExit) as e: logger.warn('Stopping the Salt Minion') + if isinstance(e, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(e)) + finally: self.shutdown() def shutdown(self): From 4061e52285b8c5ba350f069c353ecc9ce435dd96 Mon Sep 17 00:00:00 2001 From: Baiju Muthukadan Date: Thu, 7 Feb 2013 09:33:07 +0530 Subject: [PATCH 171/769] Added missing import --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6c79390..da518f1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -11,6 +11,7 @@ # that modules don't cause the build to fail from salt.version import __version__ # pylint: disable-msg=W402 from salt.utils import migrations +import salt.master try: from salt.utils import parsers From a8576877bd1ae5041c3d37a3182cc566f4e1adf2 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Sun, 10 Feb 2013 14:20:08 -0700 Subject: [PATCH 172/769] remove unneeded import --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index da518f1..6c79390 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -11,7 +11,6 @@ # that modules don't cause the build to fail from salt.version import __version__ # pylint: disable-msg=W402 from salt.utils import migrations -import salt.master try: from salt.utils import parsers From 265d2253ee1fd6355b8a445632c5f38985af0549 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 17 Feb 2013 10:56:09 +0000 Subject: [PATCH 173/769] No user interaction. --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6c79390..1495cf8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -124,6 +124,9 @@ def prepare(self): ''' self.parse_args() + import pprint + pprint.pprint(self.config) + try: if self.config['verify_env']: confd = os.path.join( From e8c9a6744659903b4a5472e7d5783c0c754044d8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 17 Feb 2013 12:56:22 +0000 Subject: [PATCH 174/769] Remove debug printing. --- src/saltext/consul/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 1495cf8..6c79390 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -124,9 +124,6 @@ def prepare(self): ''' self.parse_args() - import pprint - pprint.pprint(self.config) - try: if self.config['verify_env']: confd = os.path.join( From d4fccdda7f84aaa0a0e33307f95e2ce2d797a8fc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 28 Feb 2013 12:21:46 -0700 Subject: [PATCH 175/769] Allow for a minion to explicitly request an external pillar --- src/saltext/consul/pillar/__init__.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 67480e5..76c5553 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -32,9 +32,10 @@ class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, env): + def __init__(self, opts, grains, id_, env, ext=None): self.opts = opts self.opts['environment'] = env + self.ext = ext self.grains = grains self.id_ = id_ self.serial = salt.payload.Serial(self.opts) @@ -50,6 +51,8 @@ def compile_pillar(self): 'env': self.opts['environment'], 'ver': '2', 'cmd': '_pillar'} + if self.ext: + load['ext'] = self.ext ret = self.sreq.send('aes', self.auth.crypticle.dumps(load), 3, 7200) key = self.auth.get_keys() aes = key.private_decrypt(ret['key'], 4) @@ -61,9 +64,9 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, env): + def __init__(self, opts, grains, id_, env, ext=None): # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, env) + self.opts = self.__gen_opts(opts, grains, id_, env, ext) self.client = salt.fileclient.get_file_client(self.opts) if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -74,7 +77,7 @@ def __init__(self, opts, grains, id_, env): self.rend = salt.loader.render(self.opts, self.functions) self.ext_pillars = salt.loader.pillars(self.opts, self.functions) - def __gen_opts(self, opts_in, grains, id_, env=None): + def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): ''' The options need to be altered to conform to the file client ''' @@ -91,6 +94,11 @@ def __gen_opts(self, opts_in, grains, id_, env=None): opts['state_top'] = os.path.join('salt://', opts['state_top'][1:]) else: opts['state_top'] = os.path.join('salt://', opts['state_top']) + if ext: + if 'ext_pillar' in opts: + opts['ext_pillar'].append(ext) + else: + opts['ext_pillar'].append(ext) return opts def _get_envs(self): @@ -329,7 +337,12 @@ def ext_pillar(self, pillar): ext = self.ext_pillars[key](pillar, val) update(pillar, ext) except Exception as exc: - log.exception('Failed to load ext_pillar {0}: {1}'.format(key, exc)) + log.exception( + 'Failed to load ext_pillar {0}: {1}'.format( + key, + exc + ) + ) return pillar def compile_pillar(self): From 616c9e4eca2ef17d9d1c67787d25348b8881528c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 28 Feb 2013 14:56:33 -0500 Subject: [PATCH 176/769] Final components for explicit external pillar support --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 76c5553..8209550 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -18,14 +18,14 @@ log = logging.getLogger(__name__) -def get_pillar(opts, grains, id_, env=None): +def get_pillar(opts, grains, id_, env=None, ext=None): ''' Return the correct pillar driver based on the file_client option ''' return { 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], Pillar)(opts, grains, id_, env) + }.get(opts['file_client'], Pillar)(opts, grains, id_, env, ext) class RemotePillar(object): From 83067cb0bad81dd3b0536d2756be6cdced638dd3 Mon Sep 17 00:00:00 2001 From: Sean Channel Date: Wed, 6 Mar 2013 11:04:29 -0800 Subject: [PATCH 177/769] Setting-up message should be info instead of warn --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6c79390..fb8c98f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -71,7 +71,7 @@ def prepare(self): sys.exit(err.errno) self.setup_logfile_logger() - logger.warn('Setting up the Salt Master') + logger.info('Setting up the Salt Master') if not verify_socket(self.config['interface'], self.config['publish_port'], @@ -157,7 +157,7 @@ def prepare(self): sys.exit(err.errno) self.setup_logfile_logger() - logger.warn( + logger.info( 'Setting up the Salt Minion "{0}"'.format( self.config['id'] ) @@ -241,7 +241,7 @@ def prepare(self): sys.exit(err.errno) self.setup_logfile_logger() - logger.warn( + logger.info( 'Setting up the Salt Syndic Minion "{0}"'.format( self.config['id'] ) From c16ffc56a041a79d33a7869a281a253dd34ea1d7 Mon Sep 17 00:00:00 2001 From: Paul Traylor Date: Fri, 15 Mar 2013 11:04:32 -0700 Subject: [PATCH 178/769] Provide the master's version in pillar.data This allows you to make decisions based on the minion version and master version, for example, running pkg.install only if the salt-minion is out of date Grains would have the minion's version Pillar would have the master's version {% if grains['saltversion'] != pillar['master']['saltversion'] %} --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 67480e5..00e2481 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -15,6 +15,7 @@ from salt._compat import string_types from salt.template import compile_template from salt.utils.dictupdate import update +from salt.version import __version__ log = logging.getLogger(__name__) @@ -347,6 +348,7 @@ def compile_pillar(self): mopts.pop('grains') if 'aes' in mopts: mopts.pop('aes') + mopts['saltversion'] = __version__ pillar['master'] = mopts if errors: for error in errors: From 707530eed27a154802bddcaa94ed993c926fa20c Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Tue, 19 Mar 2013 19:41:26 -0700 Subject: [PATCH 179/769] Taking into account default_include configuration when checking for environment setup correctness --- src/saltext/consul/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fb8c98f..ad848ae 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -126,10 +126,18 @@ def prepare(self): try: if self.config['verify_env']: - confd = os.path.join( - os.path.dirname(self.config['conf_file']), - 'minion.d' - ) + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute path, + # consider it relative to folder of 'conf_file' (/etc/salt by default) + confd = os.path.join(os.path.dirname(self.config['conf_file']), confd) + else: + confd = os.path.join(os.path.dirname(self.config['conf_file']), 'minion.d') verify_env( [ self.config['pki_dir'], From d6dc5875e5d3c83201105ce18bbadd48ad2c4a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Sat, 23 Mar 2013 23:57:23 +0100 Subject: [PATCH 180/769] IPv6 address literals should be specified without brackets in config files. --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ad848ae..985beaa 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,7 +13,7 @@ from salt.utils import migrations try: - from salt.utils import parsers + from salt.utils import parsers, ip_bracket from salt.utils.verify import check_user, verify_env, verify_socket from salt.utils.verify import verify_files except ImportError as e: @@ -77,6 +77,7 @@ def prepare(self): self.config['publish_port'], self.config['ret_port']): self.exit(4, 'The ports are not available to bind\n') + self.config['interface'] = ip_bracket(self.config['interface']) migrations.migrate_paths(self.config) # Late import so logging works correctly From ea2ad327b537acf2004ce4591830ba65927761ec Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 25 Mar 2013 22:31:27 +1100 Subject: [PATCH 181/769] The pidfile is now written prior to Minion init so the process can be managed during the init phase. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ad848ae..54da7b3 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -178,8 +178,8 @@ def prepare(self): # the boot process waiting for a key to be accepted on the master. # This is the latest safe place to daemonize self.daemonize_if_required() - self.minion = salt.minion.Minion(self.config) self.set_pidfile() + self.minion = salt.minion.Minion(self.config) def start(self): ''' From ae69f84cef36a0d0b6858fa0bee3f5bbd145a096 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 28 Mar 2013 17:35:14 -0600 Subject: [PATCH 182/769] Immport salt.master to get master exit exception --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 54da7b3..9bc1204 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,6 +13,7 @@ from salt.utils import migrations try: + import salt.master from salt.utils import parsers from salt.utils.verify import check_user, verify_env, verify_socket from salt.utils.verify import verify_files From 73e6254b95aafeb24a21bdfc8c17ebf393c26ed1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 5 Apr 2013 09:23:22 -0600 Subject: [PATCH 183/769] Fix issue with MasterExit being in salt.master.py --- src/saltext/consul/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9bc1204..0401d27 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,14 +13,13 @@ from salt.utils import migrations try: - import salt.master from salt.utils import parsers from salt.utils.verify import check_user, verify_env, verify_socket from salt.utils.verify import verify_files except ImportError as e: if e.args[0] != 'No module named _msgpack': raise -from salt.exceptions import SaltSystemExit +from salt.exceptions import SaltSystemExit, MasterExit logger = logging.getLogger(__name__) @@ -100,7 +99,7 @@ def start(self): if check_user(self.config['user']): try: self.master.start() - except salt.master.MasterExit: + except MasterExit: self.shutdown() finally: sys.exit() From 83d16422f1707866da086bb14342a53eeb050d28 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sun, 7 Apr 2013 21:27:52 -0600 Subject: [PATCH 184/769] Start work on #4405 This commit adds the errors to the pillar. The next step is to have state runs fail if errors are found in the pillar. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e977cf5..c18f889 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -366,5 +366,5 @@ def compile_pillar(self): if errors: for error in errors: log.critical('Pillar render error: {0}'.format(error)) - return {} + pillar['_errors'] = errors return pillar From 2df27dc512b190d1a9b2d2077cb74bc8a9964c69 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 17 Apr 2013 02:35:45 +0100 Subject: [PATCH 185/769] Fix same behaviour as shown in saltstack/salt-cloud#378 --- src/saltext/consul/__init__.py | 39 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0401d27..7f93ad2 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -58,15 +58,11 @@ def prepare(self): pki_dir=self.config['pki_dir'], ) logfile = self.config['log_file'] - if logfile is not None and ( - not logfile.startswith('tcp://') or - not logfile.startswith('udp://') or - not logfile.startswith('file://')): + if logfile is not None and not logfile.startswith('tcp://') \ + and not logfile.startswith('udp://') \ + and not logfile.startswith('file://'): # Logfile is not using Syslog, verify - verify_files( - [logfile], - self.config['user'] - ) + verify_files([logfile], self.config['user']) except OSError as err: sys.exit(err.errno) @@ -150,17 +146,12 @@ def prepare(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - if (not self.config['log_file'].startswith('tcp://') or - not self.config['log_file'].startswith('udp://') or - not self.config['log_file'].startswith('file://')): + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith('tcp://') \ + and not logfile.startswith('udp://') \ + and not logfile.startswith('file://'): # Logfile is not using Syslog, verify - verify_files( - [self.config['log_file']], - self.config['user'] - ) - verify_files( - [self.config['log_file']], - self.config['user']) + verify_files([logfile], self.config['user']) except OSError as err: sys.exit(err.errno) @@ -237,14 +228,12 @@ def prepare(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - if (not self.config['log_file'].startswith('tcp://') or - not self.config['log_file'].startswith('udp://') or - not self.config['log_file'].startswith('file://')): + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith('tcp://') \ + and not logfile.startswith('udp://') \ + and not logfile.startswith('file://'): # Logfile is not using Syslog, verify - verify_files( - [self.config['log_file']], - self.config['user'] - ) + verify_files([logfile], self.config['user']) except OSError as err: sys.exit(err.errno) From 17c1c495b0e2defd2d3f3bad5434b91148492ee3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 18 Apr 2013 00:54:12 +0100 Subject: [PATCH 186/769] Import `salt.log` ASAP. Because we NEED to make sure that any logger instance salt instantiates is using `salt.log.SaltLoggingClass` --- src/saltext/consul/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7f93ad2..f755005 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,8 +7,14 @@ import sys import logging -# Import salt libs, the try block below bypasses an issue at build time so -# that modules don't cause the build to fail +# Import salt libs +# We import log ASAP because we NEED to make sure that any logger instance salt +# instantiates is using salt.log.SaltLoggingClass +import salt.log + + +# the try block below bypasses an issue at build time so that modules don't +# cause the build to fail from salt.version import __version__ # pylint: disable-msg=W402 from salt.utils import migrations @@ -21,7 +27,10 @@ raise from salt.exceptions import SaltSystemExit, MasterExit -logger = logging.getLogger(__name__) + +# Let's instantiate logger using salt.log.logging.getLogger() so pylint leaves +# us alone and stops complaining about an un-used import +logger = salt.log.logging.getLogger(__name__) class Master(parsers.MasterOptionParser): From bb1246b7ade6a1b5cfc7ee755e67c43994abc459 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Tue, 30 Apr 2013 19:12:24 -0700 Subject: [PATCH 187/769] fix bad & tab indentation reported by pylint --- src/saltext/consul/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 657747f..758b80c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -134,14 +134,14 @@ def prepare(self): if self.config['verify_env']: confd = self.config.get('default_include') if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute path, - # consider it relative to folder of 'conf_file' (/etc/salt by default) - confd = os.path.join(os.path.dirname(self.config['conf_file']), confd) + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute path, + # consider it relative to folder of 'conf_file' (/etc/salt by default) + confd = os.path.join(os.path.dirname(self.config['conf_file']), confd) else: confd = os.path.join(os.path.dirname(self.config['conf_file']), 'minion.d') verify_env( From fee50b90f348f685a9cbedd76da2536f23fdae46 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Wed, 1 May 2013 23:02:18 -0700 Subject: [PATCH 188/769] not X == Y -> X != Y --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c18f889..5c81bcc 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -235,7 +235,7 @@ def top_matches(self, top): matches = {} for env, body in top.items(): if self.opts['environment']: - if not env == self.opts['environment']: + if env != self.opts['environment']: continue for match, data in body.items(): if self.matcher.confirm_top( From c992e9d19c75cdf16cff337a451b13804259c6b7 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Thu, 2 May 2013 00:25:39 -0700 Subject: [PATCH 189/769] simplify / style-tweak conditionals in __init__.py-s --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5c81bcc..ecb6427 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -154,7 +154,7 @@ def get_tops(self): # Search initial top files for includes for env, ctops in tops.items(): for ctop in ctops: - if not 'include' in ctop: + if 'include' not in ctop: continue for sls in ctop['include']: include[env].append(sls) @@ -203,7 +203,7 @@ def merge_tops(self, tops): if env == 'include': continue for tgt in targets: - if not tgt in top[env]: + if tgt not in top[env]: top[env][tgt] = ctop[env][tgt] continue matches = [] From 2f3ea0e4adf7d337a5fccdbadb8344dc1baaf819 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Thu, 2 May 2013 14:21:11 -0700 Subject: [PATCH 190/769] simplify dict loops use .values() instead of .items() when not using the keys use .keys() directly instead of rebuilding lists from it --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ecb6427..718e332 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -197,7 +197,7 @@ def merge_tops(self, tops): Cleanly merge the top files ''' top = collections.defaultdict(dict) - for sourceenv, ctops in tops.items(): + for ctops in tops.values(): for ctop in ctops: for env, targets in ctop.items(): if env == 'include': From 580e1cc7eae9c02fa35db9f00c10b1c88973d225 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Mon, 6 May 2013 14:34:41 -0700 Subject: [PATCH 191/769] remove pylint pragma w/ invalid ID number ************* Module salt E0012: 18,0: Bad option value 'W402' --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 758b80c..0f20980 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -15,7 +15,7 @@ # the try block below bypasses an issue at build time so that modules don't # cause the build to fail -from salt.version import __version__ # pylint: disable-msg=W402 +from salt.version import __version__ from salt.utils import migrations try: From 32837dc17e4fcc03da9eb96fb4ebe9fffa130f2c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 May 2013 22:32:16 -0600 Subject: [PATCH 192/769] Run tune_in from the multiminion if masters is configured --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0f20980..080c0c1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -180,7 +180,10 @@ def prepare(self): # This is the latest safe place to daemonize self.daemonize_if_required() self.set_pidfile() - self.minion = salt.minion.Minion(self.config) + if isinstance(self.config.get('masters'), list): + self.minion = salt.minion.MultiMinion(self.config) + else: + self.minion = salt.minion.Minion(self.config) def start(self): ''' From b1c303ae6a68ae68ac9e1c0544d4cd535210140c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 May 2013 23:15:48 -0600 Subject: [PATCH 193/769] master instead of masters --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 080c0c1..65b7bdd 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -180,7 +180,7 @@ def prepare(self): # This is the latest safe place to daemonize self.daemonize_if_required() self.set_pidfile() - if isinstance(self.config.get('masters'), list): + if isinstance(self.config.get('master'), list): self.minion = salt.minion.MultiMinion(self.config) else: self.minion = salt.minion.Minion(self.config) From 62a9e66aeacf2474a0e85b8e1d4ecc2e68aae4d6 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 7 May 2013 22:12:00 -0600 Subject: [PATCH 194/769] Only allow certian ext_pillars to be explicitly called from minions --- src/saltext/consul/pillar/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 718e332..d30c19d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -78,6 +78,18 @@ def __init__(self, opts, grains, id_, env, ext=None): self.rend = salt.loader.render(self.opts, self.functions) self.ext_pillars = salt.loader.pillars(self.opts, self.functions) + def __valid_ext(self, ext): + ''' + Check to see if the on demand external pillar is allowed + ''' + if not isinstance(ext, dict): + return {} + valid = ['libvirt'] + for key in ext: + if key not in valid: + return {} + return ext + def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): ''' The options need to be altered to conform to the file client @@ -95,7 +107,7 @@ def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): opts['state_top'] = os.path.join('salt://', opts['state_top'][1:]) else: opts['state_top'] = os.path.join('salt://', opts['state_top']) - if ext: + if self.__valid_ext(ext): if 'ext_pillar' in opts: opts['ext_pillar'].append(ext) else: From bf50252c0d960361683cc308ecdc830e18a0b9ad Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Wed, 8 May 2013 10:59:52 -0600 Subject: [PATCH 195/769] Pillar.__valid_ext(): use any() & a set --- src/saltext/consul/pillar/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d30c19d..ae294f7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -84,10 +84,9 @@ def __valid_ext(self, ext): ''' if not isinstance(ext, dict): return {} - valid = ['libvirt'] - for key in ext: - if key not in valid: - return {} + valid = set(('libvirt',)) + if any(key not in valid for key in ext): + return {} return ext def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): From 69a76562a44162030bdde1f63b68be8822494dce Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 16 May 2013 17:35:11 -0600 Subject: [PATCH 196/769] Extend get_state to return a dict with the source And add suport to read in relative includes in states for init.sls --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ae294f7..7bf4f7f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -186,7 +186,7 @@ def get_tops(self): self.client.get_state( sls, env - ), + ).get('dest', False), self.rend, self.opts['renderer'], env=env @@ -267,7 +267,7 @@ def render_pstate(self, sls, env, mods): ''' err = '' errors = [] - fn_ = self.client.get_state(sls, env) + fn_ = self.client.get_state(sls, env).get('dest', False) if not fn_: errors.append(('Specified SLS {0} in environment {1} is not' ' available on the salt master').format(sls, env)) From 43793a31cef76e43f1a651fd695a0a5001a11d90 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Tue, 11 Jun 2013 18:17:25 -0700 Subject: [PATCH 197/769] pep8 whitespace fixes --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7bf4f7f..a6588b5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -107,7 +107,7 @@ def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): else: opts['state_top'] = os.path.join('salt://', opts['state_top']) if self.__valid_ext(ext): - if 'ext_pillar' in opts: + if 'ext_pillar' in opts: opts['ext_pillar'].append(ext) else: opts['ext_pillar'].append(ext) From 0d00482a7f2cd5335f6689bd66000fc1773dce09 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Wed, 26 Jun 2013 19:31:55 -0700 Subject: [PATCH 198/769] add ability to pass defaults to included pillar sls --- src/saltext/consul/pillar/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a6588b5..33ce37c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -261,7 +261,7 @@ def top_matches(self, top): matches[env].append(item) return matches - def render_pstate(self, sls, env, mods): + def render_pstate(self, sls, env, mods, defaults={}): ''' Collect a single pillar sls file and render it ''' @@ -274,7 +274,7 @@ def render_pstate(self, sls, env, mods): state = None try: state = compile_template( - fn_, self.rend, self.opts['renderer'], env, sls) + fn_, self.rend, self.opts['renderer'], env, sls, **defaults) except Exception as exc: errors.append(('Rendering SLS {0} failed, render error:\n{1}' .format(sls, exc))) @@ -292,11 +292,15 @@ def render_pstate(self, sls, env, mods): errors.append(err) else: for sub_sls in state.pop('include'): + if isinstance(sub_sls, dict): + sub_sls, v = sub_sls.iteritems().next() + defaults = v.get('defaults', {}) if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, env, - mods + mods, + defaults ) if nstate: state.update(nstate) From 3ea4f36f10db59e0d8ccae8aa7dcebac8e0a05d2 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Fri, 28 Jun 2013 13:16:13 -0700 Subject: [PATCH 199/769] pillar: allow include to provide a key within which to next --- src/saltext/consul/pillar/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 33ce37c..909a6a1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -295,6 +295,7 @@ def render_pstate(self, sls, env, mods, defaults={}): if isinstance(sub_sls, dict): sub_sls, v = sub_sls.iteritems().next() defaults = v.get('defaults', {}) + key = v.get('key', {}) if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, @@ -303,7 +304,10 @@ def render_pstate(self, sls, env, mods, defaults={}): defaults ) if nstate: - state.update(nstate) + if key: + state[key] = nstate + else: + state.update(nstate) if err: errors += err return state, mods, errors From 2ef6b2d741d7642bd75eb0734a305542f26d2aa8 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Fri, 28 Jun 2013 13:19:16 -0700 Subject: [PATCH 200/769] pillar: allow include to provide a key within which to nest -- bugfix --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 909a6a1..30e46f3 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -295,7 +295,7 @@ def render_pstate(self, sls, env, mods, defaults={}): if isinstance(sub_sls, dict): sub_sls, v = sub_sls.iteritems().next() defaults = v.get('defaults', {}) - key = v.get('key', {}) + key = v.get('key', None) if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, From b7aaf10eecd9186a5d96c066c9272fc03ed1233b Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Fri, 28 Jun 2013 13:41:06 -0700 Subject: [PATCH 201/769] pillar: allow include to provide a key within which to nest -- another bugfix --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 30e46f3..a2031cf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -296,6 +296,8 @@ def render_pstate(self, sls, env, mods, defaults={}): sub_sls, v = sub_sls.iteritems().next() defaults = v.get('defaults', {}) key = v.get('key', None) + else: + key = None if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, From 41838d22d179bb0fb3208f179ba753171e04de03 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Mon, 1 Jul 2013 16:07:19 -0700 Subject: [PATCH 202/769] Pillar.render_pstate: avoid mutable default argument value (pylint W0102) --- src/saltext/consul/pillar/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a2031cf..7b0cc3b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -261,10 +261,12 @@ def top_matches(self, top): matches[env].append(item) return matches - def render_pstate(self, sls, env, mods, defaults={}): + def render_pstate(self, sls, env, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' + if defaults is None: + defaults = {} err = '' errors = [] fn_ = self.client.get_state(sls, env).get('dest', False) From e94b69df7c6c2db1ff55843dad17084e26904eec Mon Sep 17 00:00:00 2001 From: "martin f. krafft" Date: Fri, 28 Jun 2013 07:41:20 +0200 Subject: [PATCH 203/769] Pass minion_id to ext_pillar Pillar data is always targetted at a minion, so it makes sense to actually pass the minion ID via the ext_pillar interface, rather than expecting plugins to obtain it from __opts__. While this patch also changes all existing plugins, it concedes that there are going to be plugins outside of the tree, continuing to use the old interface. Such plugins will trigger a warning written to the master log. This warning can later be made critical as part of proper deprecation. Signed-off-by: martin f. krafft --- src/saltext/consul/pillar/__init__.py | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7b0cc3b..ce529c2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -353,13 +353,33 @@ def ext_pillar(self, pillar): log.critical(err) continue try: - if isinstance(val, dict): - ext = self.ext_pillars[key](pillar, **val) - elif isinstance(val, list): - ext = self.ext_pillars[key](pillar, *val) - else: - ext = self.ext_pillars[key](pillar, val) - update(pillar, ext) + try: + # try the new interface, which includes the minion ID + # as first argument + if isinstance(val, dict): + ext = self.ext_pillars[key](self.opts['id'], pillar, **val) + elif isinstance(val, list): + ext = self.ext_pillars[key](self.opts['id'], pillar, *val) + else: + ext = self.ext_pillars[key](self.opts['id'], pillar, val) + update(pillar, ext) + + except TypeError, e: + if e.message.startswith('ext_pillar() takes exactly '): + log.warning('Deprecation warning: ext_pillar "{0}"'\ + ' needs to accept minion_id as first'\ + ' argument'.format(key)) + else: + raise + + if isinstance(val, dict): + ext = self.ext_pillars[key](pillar, **val) + elif isinstance(val, list): + ext = self.ext_pillars[key](pillar, *val) + else: + ext = self.ext_pillars[key](pillar, val) + update(pillar, ext) + except Exception as exc: log.exception( 'Failed to load ext_pillar {0}: {1}'.format( From b08798294113130b4ab94aa7e3e058d68e678150 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Wed, 3 Jul 2013 12:29:40 -0700 Subject: [PATCH 204/769] pillar/__init__.py: use modern "except X as Y" syntax --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ce529c2..e6943cf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -364,7 +364,7 @@ def ext_pillar(self, pillar): ext = self.ext_pillars[key](self.opts['id'], pillar, val) update(pillar, ext) - except TypeError, e: + except TypeError as e: if e.message.startswith('ext_pillar() takes exactly '): log.warning('Deprecation warning: ext_pillar "{0}"'\ ' needs to accept minion_id as first'\ From ec0d54e5ac8ef84188617166143a3c1bcd86c157 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Fri, 12 Jul 2013 23:53:59 +0000 Subject: [PATCH 205/769] pillar: fix minor bug --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e6943cf..eae6d1d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -110,7 +110,7 @@ def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): if 'ext_pillar' in opts: opts['ext_pillar'].append(ext) else: - opts['ext_pillar'].append(ext) + opts['ext_pillar'] = [ext] return opts def _get_envs(self): @@ -391,7 +391,7 @@ def ext_pillar(self, pillar): def compile_pillar(self): ''' - Render the pillar dta and return + Render the pillar data and return ''' top, terrors = self.get_top() matches = self.top_matches(top) From 2b3db6a8628102b8a80f821c212009b35ea6957c Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Sat, 13 Jul 2013 20:10:04 +0000 Subject: [PATCH 206/769] wip: preapprove virt minion keys --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index eae6d1d..b14b62f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -84,7 +84,7 @@ def __valid_ext(self, ext): ''' if not isinstance(ext, dict): return {} - valid = set(('libvirt',)) + valid = set(('libvirt', 'virtkey')) if any(key not in valid for key in ext): return {} return ext From 24aa759082278ff5edd52d5e3ad941d4c2f33e1e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 20 Jul 2013 16:06:16 +0100 Subject: [PATCH 207/769] **ALL** Salt `DeprecationWarning`'s should be shown **once**. --- src/saltext/consul/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 65b7bdd..8d82157 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,6 +6,15 @@ import os import sys import logging +import warnings + +# All salt related deprecation warnings should be shown once each! +warnings.filterwarnings( + 'once', # Show once + '', # No deprecation message match + DeprecationWarning, # This filter is for DeprecationWarnings + r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' +) # Import salt libs # We import log ASAP because we NEED to make sure that any logger instance salt From 856461ff94199ce08bb181cbd695ff58ac063ba6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 24 Jul 2013 22:51:35 -0500 Subject: [PATCH 208/769] Fix minion traceback when pillar SLS is malformed When pillar data is rendered for use by pillar.get, SLS that does not render to a dictionary results in a traceback on the minion. This commit resolves that traceback. Fixes #5910. --- src/saltext/consul/pillar/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b14b62f..168b169 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -19,6 +19,7 @@ log = logging.getLogger(__name__) + def get_pillar(opts, grains, id_, env=None, ext=None): ''' Return the correct pillar driver based on the file_client option @@ -328,7 +329,10 @@ def render_pillar(self, matches): for sls in pstates: pstate, mods, err = self.render_pstate(sls, env, mods) if pstate: - pillar.update(pstate) + try: + pillar.update(pstate) + except: + pass if err: errors += err return pillar, errors @@ -366,8 +370,8 @@ def ext_pillar(self, pillar): except TypeError as e: if e.message.startswith('ext_pillar() takes exactly '): - log.warning('Deprecation warning: ext_pillar "{0}"'\ - ' needs to accept minion_id as first'\ + log.warning('Deprecation warning: ext_pillar "{0}"' + ' needs to accept minion_id as first' ' argument'.format(key)) else: raise From 84570f8cf99a9a6cb28c0c7f54ae6fd9ad824018 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Thu, 25 Jul 2013 15:27:24 -0700 Subject: [PATCH 209/769] pillar: specify explicit exception type in `except` clause ************* Module salt.pillar W0702:334,20:Pillar.render_pillar: No exception type(s) specified --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 168b169..69e82d1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -331,7 +331,7 @@ def render_pillar(self, matches): if pstate: try: pillar.update(pstate) - except: + except Exception: pass if err: errors += err From 1cf4da7c837b6fa28bd1578705a6da873442e47c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 25 Jul 2013 13:50:59 +0100 Subject: [PATCH 210/769] No general exception catching while rendering pillar. Refs #6305. Refs #5910. * All errors while processing pillar data are now logged besides being returned. * Removed the general exception catch. All we need to is check for the expected data type. If it ain't, log the error and continue to the next iteration. * Added a mocked test case which besides testing the issue reported on #5910, also tests for the proper includes data format and if salt fails accordingly and even if common data is merged. --- src/saltext/consul/pillar/__init__.py | 46 +++++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 69e82d1..80734d7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -272,27 +272,34 @@ def render_pstate(self, sls, env, mods, defaults=None): errors = [] fn_ = self.client.get_state(sls, env).get('dest', False) if not fn_: - errors.append(('Specified SLS {0} in environment {1} is not' - ' available on the salt master').format(sls, env)) + msg = ('Specified SLS {0!r} in environment {1!r} is not' + ' available on the salt master').format(sls, env) + log.error(msg) + errors.append(msg) state = None try: state = compile_template( fn_, self.rend, self.opts['renderer'], env, sls, **defaults) except Exception as exc: - errors.append(('Rendering SLS {0} failed, render error:\n{1}' - .format(sls, exc))) + msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( + sls, exc + ) + log.error(msg) + errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): - errors.append(('SLS {0} does not render to a dictionary' - .format(sls))) + msg = 'SLS {0!r} does not render to a dictionary'.format(sls) + log.error(msg) + errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): - err = ('Include Declaration in SLS {0} is not formed ' - 'as a list'.format(sls)) - errors.append(err) + msg = ('Include Declaration in SLS {0!r} is not ' + 'formed as a list'.format(sls)) + log.error(msg) + errors.append(msg) else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): @@ -328,13 +335,24 @@ def render_pillar(self, matches): mods = set() for sls in pstates: pstate, mods, err = self.render_pstate(sls, env, mods) - if pstate: - try: - pillar.update(pstate) - except Exception: - pass + if err: errors += err + + if pstate is not None: + if not isinstance(pstate, dict): + log.error( + 'The rendered pillar sls file, {0!r} state did ' + 'not return the expected data format. This is ' + 'a sign of a malformed pillar sls file. Returned ' + 'errors: {1}'.format( + sls, + ', '.join(['{0!r}'.format(e) for e in errors]) + ) + ) + continue + pillar.update(pstate) + return pillar, errors def ext_pillar(self, pillar): From 2bcbd409211600f3df5b42d20b7031bb2ce140ec Mon Sep 17 00:00:00 2001 From: Luper Rouch Date: Wed, 7 Aug 2013 22:50:24 +0200 Subject: [PATCH 211/769] added support for order clauses in pillar top files Conflicts: tests/unit/pillar_test.py --- src/saltext/consul/pillar/__init__.py | 33 ++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 80734d7..fe8b568 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -15,6 +15,7 @@ from salt._compat import string_types from salt.template import compile_template from salt.utils.dictupdate import update +from salt.utils.odict import OrderedDict from salt.version import __version__ log = logging.getLogger(__name__) @@ -209,25 +210,45 @@ def merge_tops(self, tops): Cleanly merge the top files ''' top = collections.defaultdict(dict) + orders = collections.defaultdict(dict) for ctops in tops.values(): for ctop in ctops: for env, targets in ctop.items(): if env == 'include': continue for tgt in targets: - if tgt not in top[env]: - top[env][tgt] = ctop[env][tgt] - continue matches = [] states = set() - for comp in top[env][tgt]: + orders[env][tgt] = 0 + for comp in ctop[env][tgt]: if isinstance(comp, dict): - matches.append(comp) + if 'match' in comp: + matches.append(comp) + if 'order' in comp: + order = comp['order'] + if not isinstance(order, int): + try: + order = int(order) + except ValueError: + order = 0 + orders[env][tgt] = order if isinstance(comp, string_types): states.add(comp) top[env][tgt] = matches top[env][tgt].extend(list(states)) - return top + return self.sort_top_targets(top, orders) + + def sort_top_targets(self, top, orders): + ''' + Returns the sorted high data from the merged top files + ''' + sorted_top = collections.defaultdict(OrderedDict) + for env, targets in top.items(): + sorted_targets = sorted(targets.keys(), + key=lambda target: orders[env][target]) + for target in sorted_targets: + sorted_top[env][target] = targets[target] + return sorted_top def get_top(self): ''' From 34924bccad116585857c34976c1a59a85cf63c5d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 10 Aug 2013 23:26:48 -0600 Subject: [PATCH 212/769] more whitespace --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fe8b568..4c4e339 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -244,11 +244,11 @@ def sort_top_targets(self, top, orders): ''' sorted_top = collections.defaultdict(OrderedDict) for env, targets in top.items(): - sorted_targets = sorted(targets.keys(), + sorted_targets = sorted(targets.keys(), key=lambda target: orders[env][target]) for target in sorted_targets: sorted_top[env][target] = targets[target] - return sorted_top + return sorted_top def get_top(self): ''' From 6d02f2d810a953221b3df860e6e76788600b142c Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Sat, 10 Aug 2013 23:34:26 -0700 Subject: [PATCH 213/769] assorted PEP 8 fixes; mostly inline whitespace --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fe8b568..4c4e339 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -244,11 +244,11 @@ def sort_top_targets(self, top, orders): ''' sorted_top = collections.defaultdict(OrderedDict) for env, targets in top.items(): - sorted_targets = sorted(targets.keys(), + sorted_targets = sorted(targets.keys(), key=lambda target: orders[env][target]) for target in sorted_targets: sorted_top[env][target] = targets[target] - return sorted_top + return sorted_top def get_top(self): ''' From 1ed469d846c83517faf4b15cf597d0571c6205ef Mon Sep 17 00:00:00 2001 From: David Boucha Date: Mon, 26 Aug 2013 16:03:26 -0600 Subject: [PATCH 214/769] Send {} if grains is None --- src/saltext/consul/pillar/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4c4e339..d4c9b19 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -98,7 +98,10 @@ def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' - opts['grains'] = grains + if not grains: + opts['grains'] = {} + else: + opts['grains'] = grains opts['id'] = id_ if 'environment' not in opts: opts['environment'] = env From 19173cc9bd82173e39e54d84e81f2ad7b3ec0849 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 18 Jul 2013 20:14:08 +0100 Subject: [PATCH 215/769] `salt.log` is now a package instead of a module. The necessary changes were made to assure backwards compatibility. --- src/saltext/consul/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8d82157..7228967 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -18,8 +18,8 @@ # Import salt libs # We import log ASAP because we NEED to make sure that any logger instance salt -# instantiates is using salt.log.SaltLoggingClass -import salt.log +# instantiates is using salt.log.config.SaltLoggingClass +import salt.log.config # the try block below bypasses an issue at build time so that modules don't @@ -37,9 +37,9 @@ from salt.exceptions import SaltSystemExit, MasterExit -# Let's instantiate logger using salt.log.logging.getLogger() so pylint leaves -# us alone and stops complaining about an un-used import -logger = salt.log.logging.getLogger(__name__) +# Let's instantiate logger using salt.log.config.logging.getLogger() so pylint +# leaves us alone and stops complaining about an un-used import +logger = salt.log.config.logging.getLogger(__name__) class Master(parsers.MasterOptionParser): From a6c0f54e36367bc946e10fd3d2829230ae8161c2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 19 Jul 2013 16:33:45 +0100 Subject: [PATCH 216/769] Renamed `salt.log.config` to `salt.log.setup`. --- src/saltext/consul/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7228967..b798e63 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -18,8 +18,8 @@ # Import salt libs # We import log ASAP because we NEED to make sure that any logger instance salt -# instantiates is using salt.log.config.SaltLoggingClass -import salt.log.config +# instantiates is using salt.log.setup.SaltLoggingClass +import salt.log.setup # the try block below bypasses an issue at build time so that modules don't @@ -37,9 +37,9 @@ from salt.exceptions import SaltSystemExit, MasterExit -# Let's instantiate logger using salt.log.config.logging.getLogger() so pylint +# Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint # leaves us alone and stops complaining about an un-used import -logger = salt.log.config.logging.getLogger(__name__) +logger = salt.log.setup.logging.getLogger(__name__) class Master(parsers.MasterOptionParser): From 2e513fbcaaa920ca4a5732af8db82ef1d2ff8877 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 20 Jul 2013 22:43:03 +0100 Subject: [PATCH 217/769] Some formatting fixes. --- src/saltext/consul/__init__.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b798e63..cdbfc1e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -5,7 +5,6 @@ # Import python libs import os import sys -import logging import warnings # All salt related deprecation warnings should be shown once each! @@ -31,8 +30,8 @@ from salt.utils import parsers, ip_bracket from salt.utils.verify import check_user, verify_env, verify_socket from salt.utils.verify import verify_files -except ImportError as e: - if e.args[0] != 'No module named _msgpack': +except ImportError as exc: + if exc.args[0] != 'No module named _msgpack': raise from salt.exceptions import SaltSystemExit, MasterExit @@ -148,11 +147,16 @@ def prepare(self): # Value is of the form "minion.d/*.conf" confd = os.path.dirname(confd) if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute path, - # consider it relative to folder of 'conf_file' (/etc/salt by default) - confd = os.path.join(os.path.dirname(self.config['conf_file']), confd) + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) + confd = os.path.join( + os.path.dirname(self.config['conf_file']), confd + ) else: - confd = os.path.join(os.path.dirname(self.config['conf_file']), 'minion.d') + confd = os.path.join( + os.path.dirname(self.config['conf_file']), 'minion.d' + ) verify_env( [ self.config['pki_dir'], @@ -208,12 +212,12 @@ def start(self): try: if check_user(self.config['user']): self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as e: + except (KeyboardInterrupt, SaltSystemExit) as exc: logger.warn('Stopping the Salt Minion') - if isinstance(e, KeyboardInterrupt): + if isinstance(exc, KeyboardInterrupt): logger.warn('Exiting on Ctrl-c') else: - logger.error(str(e)) + logger.error(str(exc)) finally: self.shutdown() From a077b2edcd62a20797906fc13d7d3c2bb9e18f63 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 19 Sep 2013 00:16:38 +0100 Subject: [PATCH 218/769] Comply to PEP-263. --- src/saltext/consul/__init__.py | 1 + src/saltext/consul/pillar/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cdbfc1e..b8b4b1e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- ''' Make me some salt! ''' diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d4c9b19..98523e9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- ''' Render the pillar data ''' From 4bfc3d11971240a6bb7f884b52b1d3b4f8fb8494 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 18 Sep 2013 22:19:41 -0600 Subject: [PATCH 219/769] Add encoding to modules package --- src/saltext/consul/modules/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/modules/__init__.py b/src/saltext/consul/modules/__init__.py index e69de29..0d649bc 100644 --- a/src/saltext/consul/modules/__init__.py +++ b/src/saltext/consul/modules/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +''' +Execution Module Directory +''' From 82b022550d0dac0ae5a032520f7e9a48e80b22e4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 18 Sep 2013 22:22:43 -0600 Subject: [PATCH 220/769] Add encoding and docstring to states dir --- src/saltext/consul/states/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/states/__init__.py b/src/saltext/consul/states/__init__.py index e69de29..fc89223 100644 --- a/src/saltext/consul/states/__init__.py +++ b/src/saltext/consul/states/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +''' +States Directory +''' From 5c6d4103f592af9339ac7042a884af664ff55089 Mon Sep 17 00:00:00 2001 From: K Jonathan Harker Date: Thu, 19 Sep 2013 19:36:27 +0000 Subject: [PATCH 221/769] Render errors are critical --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 98523e9..951527c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -309,7 +309,7 @@ def render_pstate(self, sls, env, mods, defaults=None): msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc ) - log.error(msg) + log.critical(msg) errors.append(msg) mods.add(sls) nstate = None From ad7d36ef68adfca92324108e01530968ecfaca03 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 20 Sep 2013 12:54:52 +0100 Subject: [PATCH 222/769] Fix `pep8` `E271` issues. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 951527c..1326511 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -385,7 +385,7 @@ def ext_pillar(self, pillar): Render the external pillar data ''' if not 'ext_pillar' in self.opts: - return {} + return {} if not isinstance(self.opts['ext_pillar'], list): log.critical('The "ext_pillar" option is malformed') return {} From b5323637fa42efb039b5f123ae6e26338a3fdf76 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 10 Oct 2013 15:12:00 -0600 Subject: [PATCH 223/769] Restore the pillar[master][file_roots] when returning the pillar --- src/saltext/consul/pillar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 1326511..b32abfe 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -69,6 +69,8 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, id_, env, ext=None): + # Store the file_roots path so we can restore later. Issue 5449 + self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, id_, env, ext) self.client = salt.fileclient.get_file_client(self.opts) @@ -451,6 +453,8 @@ def compile_pillar(self): mopts.pop('grains') if 'aes' in mopts: mopts.pop('aes') + # Restore the actual file_roots path. Issue 5449 + mopts['file_roots'] = self.actual_file_roots mopts['saltversion'] = __version__ pillar['master'] = mopts if errors: From 0dd57d63499eba8f5db59ca65e668726b82aa45e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 11 Oct 2013 11:53:00 -0600 Subject: [PATCH 224/769] Fix issue 5951 - correct __opts__['file_roots'] in ext_pillars --- src/saltext/consul/pillar/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b32abfe..ea96b97 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -81,7 +81,11 @@ def __init__(self, opts, grains, id_, env, ext=None): self.functions = salt.loader.minion_mods(self.opts) self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) - self.ext_pillars = salt.loader.pillars(self.opts, self.functions) + # Fix self.opts['file_roots'] so that ext_pillars know the real + # location of file_roots. Issue 5951 + ext_pillar_opts = dict(self.opts) + ext_pillar_opts['file_roots'] = self.actual_file_roots + self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) def __valid_ext(self, ext): ''' From 2b3cc7fdb37f1f52fa64e176765688b47aa8968c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 15 Oct 2013 22:09:37 +0100 Subject: [PATCH 225/769] Fix syslog logic checks for `file://`, `tcp://` and `udp://`. Fixes #7754. --- src/saltext/consul/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b8b4b1e..3176569 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -76,9 +76,9 @@ def prepare(self): pki_dir=self.config['pki_dir'], ) logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith('tcp://') \ - and not logfile.startswith('udp://') \ - and not logfile.startswith('file://'): + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: @@ -171,9 +171,9 @@ def prepare(self): pki_dir=self.config['pki_dir'], ) logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith('tcp://') \ - and not logfile.startswith('udp://') \ - and not logfile.startswith('file://'): + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: @@ -256,9 +256,9 @@ def prepare(self): pki_dir=self.config['pki_dir'], ) logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith('tcp://') \ - and not logfile.startswith('udp://') \ - and not logfile.startswith('file://'): + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: From 9bdc6dee41f1f5f52b822e38d55d355d2adebba7 Mon Sep 17 00:00:00 2001 From: Xavier Barbosa Date: Tue, 29 Oct 2013 10:45:11 +0100 Subject: [PATCH 226/769] Hide with-statements warnings --- src/saltext/consul/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3176569..8985f5d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -16,6 +16,13 @@ r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' ) +# While we are supporting Python2.6, hide nested with-statements warnings +warnings.filterwarnings( + 'ignore', + 'With-statements now directly support multiple context managers', + DeprecationWarning +) + # Import salt libs # We import log ASAP because we NEED to make sure that any logger instance salt # instantiates is using salt.log.setup.SaltLoggingClass From f5fbe9562baee3bdb1c82f7410dcfaf72c0be344 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 9 Nov 2013 12:17:43 +0000 Subject: [PATCH 227/769] Deprecate `env` usage in favor of `saltenv`. --- src/saltext/consul/pillar/__init__.py | 114 +++++++++++++++----------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ea96b97..305b756 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -22,23 +22,32 @@ log = logging.getLogger(__name__) -def get_pillar(opts, grains, id_, env=None, ext=None): +def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): ''' Return the correct pillar driver based on the file_client option ''' + if env is not None: + salt.utils.warn_until( + 'Boron', + 'Passing a salt environment should be done using \'saltenv\' ' + 'not \'env\'. This functionality will be removed in Salt Boron.' + ) + # Backwards compatibility + saltenv = env + return { 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], Pillar)(opts, grains, id_, env, ext) + }.get(opts['file_client'], Pillar)(opts, grains, id_, saltenv, ext) class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, env, ext=None): + def __init__(self, opts, grains, id_, saltenv, ext=None): self.opts = opts - self.opts['environment'] = env + self.opts['environment'] = saltenv self.ext = ext self.grains = grains self.id_ = id_ @@ -52,7 +61,7 @@ def compile_pillar(self): ''' load = {'id': self.id_, 'grains': self.grains, - 'env': self.opts['environment'], + 'saltenv': self.opts['environment'], 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -68,11 +77,11 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, env, ext=None): + def __init__(self, opts, grains, id_, saltenv, ext=None): # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, env, ext) + self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) self.client = salt.fileclient.get_file_client(self.opts) if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -98,10 +107,19 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): + def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): ''' The options need to be altered to conform to the file client ''' + if env is not None: + salt.utils.warn_until( + 'Boron', + 'Passing a salt environment should be done using \'saltenv\' ' + 'not \'env\'. This functionality will be removed in Salt ' + 'Boron.' + ) + # Backwards compatibility + saltenv = env opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' @@ -111,7 +129,7 @@ def __gen_opts(self, opts_in, grains, id_, env=None, ext=None): opts['grains'] = grains opts['id'] = id_ if 'environment' not in opts: - opts['environment'] = env + opts['environment'] = saltenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): @@ -157,16 +175,16 @@ def get_tops(self): ) ] else: - for env in self._get_envs(): - tops[env].append( + for saçtenv in self._get_envs(): + tops[saltenv].append( compile_template( self.client.cache_file( self.opts['state_top'], - env + saltenv ), self.rend, self.opts['renderer'], - env=env + saltenv=saltenv ) ) except Exception as exc: @@ -175,43 +193,43 @@ def get_tops(self): .format(exc))) # Search initial top files for includes - for env, ctops in tops.items(): + for saltenv, ctops in tops.items(): for ctop in ctops: if 'include' not in ctop: continue for sls in ctop['include']: - include[env].append(sls) + include[saltenv].append(sls) ctop.pop('include') # Go through the includes and pull out the extra tops and add them while include: pops = [] - for env, states in include.items(): - pops.append(env) + for saltenv, states in include.items(): + pops.append(saltenv) if not states: continue for sls in states: - if sls in done[env]: + if sls in done[saltenv]: continue try: - tops[env].append( + tops[satenv].append( compile_template( self.client.get_state( sls, - env + saltenv ).get('dest', False), self.rend, self.opts['renderer'], - env=env + saltenv=saltenv ) ) except Exception as exc: errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) - done[env].append(sls) - for env in pops: - if env in include: - include.pop(env) + done[saltenv].append(sls) + for saltenv in pops: + if saltenv in include: + include.pop(saltenv) return tops, errors @@ -223,14 +241,14 @@ def merge_tops(self, tops): orders = collections.defaultdict(dict) for ctops in tops.values(): for ctop in ctops: - for env, targets in ctop.items(): - if env == 'include': + for saltenv, targets in ctop.items(): + if saltenv == 'include': continue for tgt in targets: matches = [] states = set() - orders[env][tgt] = 0 - for comp in ctop[env][tgt]: + orders[saltenv][tgt] = 0 + for comp in ctop[saltenv][tgt]: if isinstance(comp, dict): if 'match' in comp: matches.append(comp) @@ -241,11 +259,11 @@ def merge_tops(self, tops): order = int(order) except ValueError: order = 0 - orders[env][tgt] = order + orders[saltenv][tgt] = order if isinstance(comp, string_types): states.add(comp) - top[env][tgt] = matches - top[env][tgt].extend(list(states)) + top[saltenv][tgt] = matches + top[saltenv][tgt].extend(list(states)) return self.sort_top_targets(top, orders) def sort_top_targets(self, top, orders): @@ -253,11 +271,11 @@ def sort_top_targets(self, top, orders): Returns the sorted high data from the merged top files ''' sorted_top = collections.defaultdict(OrderedDict) - for env, targets in top.items(): + for saltenv, targets in top.items(): sorted_targets = sorted(targets.keys(), - key=lambda target: orders[env][target]) + key=lambda target: orders[saltenv][target]) for target in sorted_targets: - sorted_top[env][target] = targets[target] + sorted_top[saltenv][target] = targets[target] return sorted_top def get_top(self): @@ -273,12 +291,12 @@ def top_matches(self, top): that this minion needs to execute. Returns: - {'env': ['state1', 'state2', ...]} + {'saltenv': ['state1', 'state2', ...]} ''' matches = {} - for env, body in top.items(): + for saltenv, body in top.items(): if self.opts['environment']: - if env != self.opts['environment']: + if saltenv != self.opts['environment']: continue for match, data in body.items(): if self.matcher.confirm_top( @@ -286,14 +304,14 @@ def top_matches(self, top): data, self.opts.get('nodegroups', {}), ): - if env not in matches: - matches[env] = [] + if saltenv not in matches: + matches[saltenv] = [] for item in data: if isinstance(item, string_types): - matches[env].append(item) + matches[saltenv].append(item) return matches - def render_pstate(self, sls, env, mods, defaults=None): + def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' @@ -301,16 +319,16 @@ def render_pstate(self, sls, env, mods, defaults=None): defaults = {} err = '' errors = [] - fn_ = self.client.get_state(sls, env).get('dest', False) + fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: msg = ('Specified SLS {0!r} in environment {1!r} is not' - ' available on the salt master').format(sls, env) + ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) state = None try: state = compile_template( - fn_, self.rend, self.opts['renderer'], env, sls, **defaults) + fn_, self.rend, self.opts['renderer'], saltenv, sls, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc @@ -342,7 +360,7 @@ def render_pstate(self, sls, env, mods, defaults=None): if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, - env, + saltenv, mods, defaults ) @@ -362,10 +380,10 @@ def render_pillar(self, matches): ''' pillar = {} errors = [] - for env, pstates in matches.items(): + for saltenv, pstates in matches.items(): mods = set() for sls in pstates: - pstate, mods, err = self.render_pstate(sls, env, mods) + pstate, mods, err = self.render_pstate(sls, saltenv, mods) if err: errors += err From 3e1fa19f8ee4bbe94fa2b3cd025f17ce5c94c82d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 9 Nov 2013 12:32:05 +0000 Subject: [PATCH 228/769] Fix typo. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 305b756..0013b8e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -175,7 +175,7 @@ def get_tops(self): ) ] else: - for saçtenv in self._get_envs(): + for saltenv in self._get_envs(): tops[saltenv].append( compile_template( self.client.cache_file( From 15dc00a8bdcf1623786f39a5930960031f9f81bb Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Tue, 12 Nov 2013 10:54:57 -0800 Subject: [PATCH 229/769] fix typo in pillar/__init__.py salt/pillar/__init__.py:214: [E0602(undefined-variable), Pillar.get_tops] Undefined variable 'satenv' --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0013b8e..9c54e46 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -211,7 +211,7 @@ def get_tops(self): if sls in done[saltenv]: continue try: - tops[satenv].append( + tops[saltenv].append( compile_template( self.client.get_state( sls, From eae0d34b6d07f74e01f7035e3b12894c394311a3 Mon Sep 17 00:00:00 2001 From: Heewa Barfchin Date: Fri, 15 Nov 2013 17:43:05 -0500 Subject: [PATCH 230/769] Verify pillar from master on minion. In some cases, it's possible for minion to get an invalid pillar from master, through a mostly valid reply. Check that at least it's a dict, otherwise later code tries to do lookups in it and fails confusingly. --- src/saltext/consul/pillar/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 9c54e46..8cc6647 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -70,7 +70,14 @@ def compile_pillar(self): key = self.auth.get_keys() aes = key.private_decrypt(ret['key'], 4) pcrypt = salt.crypt.Crypticle(self.opts, aes) - return pcrypt.loads(ret['pillar']) + ret_pillar = pcrypt.loads(ret['pillar']) + if not isinstance(ret_pillar, dict): + log.error( + 'Got a bad pillar from master, type {0}, expecting dict: ' + '{1}'.format(type(ret_pillar).__name__, ret_pillar) + ) + return {} + return ret_pillar class Pillar(object): From 88f058c5a1c66aa9988c403e9f72a93003707baf Mon Sep 17 00:00:00 2001 From: yannj-fr Date: Wed, 11 Dec 2013 20:24:50 +0100 Subject: [PATCH 231/769] Update recursively pillar when reading from several SLS files instead of a simple dict update --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8cc6647..971dd1d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -407,7 +407,7 @@ def render_pillar(self, matches): ) ) continue - pillar.update(pstate) + update(pillar, pstate) return pillar, errors From 90de6438bc65a4358c8c1efd6ffc0ebae1cbd7d8 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 13 Dec 2013 16:56:48 -0700 Subject: [PATCH 232/769] Complete abstraction of ZeroMQ. --- src/saltext/consul/pillar/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 971dd1d..3430210 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -13,12 +13,14 @@ import salt.fileclient import salt.minion import salt.crypt +import salt.transport from salt._compat import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.odict import OrderedDict from salt.version import __version__ + log = logging.getLogger(__name__) @@ -52,8 +54,8 @@ def __init__(self, opts, grains, id_, saltenv, ext=None): self.grains = grains self.id_ = id_ self.serial = salt.payload.Serial(self.opts) - self.sreq = salt.payload.SREQ(self.opts['master_uri']) - self.auth = salt.crypt.SAuth(opts) + self.sreq = salt.transport.Channel.factory(opts) + # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): ''' @@ -66,11 +68,14 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret = self.sreq.send('aes', self.auth.crypticle.dumps(load), 3, 7200) - key = self.auth.get_keys() - aes = key.private_decrypt(ret['key'], 4) - pcrypt = salt.crypt.Crypticle(self.opts, aes) - ret_pillar = pcrypt.loads(ret['pillar']) + # ret = self.sreq.send(load, tries=3, timeout=7200) + ret_pillar = self.sreq.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) + + # key = self.auth.get_keys() + # aes = key.private_decrypt(ret['key'], 4) + # pcrypt = salt.crypt.Crypticle(self.opts, aes) + # ret_pillar = pcrypt.loads(ret['pillar']) + if not isinstance(ret_pillar, dict): log.error( 'Got a bad pillar from master, type {0}, expecting dict: ' From 8831d48c0c6c9ad4108b07bc8f081e44dcd6bdbc Mon Sep 17 00:00:00 2001 From: Vahid Hamidullah Date: Mon, 16 Dec 2013 16:50:21 -0500 Subject: [PATCH 233/769] Fixes pillar namespace merging not preserving order when replacing keys --- src/saltext/consul/pillar/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 971dd1d..20f4ca7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -253,7 +253,7 @@ def merge_tops(self, tops): continue for tgt in targets: matches = [] - states = set() + states = OrderedDict(); orders[saltenv][tgt] = 0 for comp in ctop[saltenv][tgt]: if isinstance(comp, dict): @@ -268,9 +268,9 @@ def merge_tops(self, tops): order = 0 orders[saltenv][tgt] = order if isinstance(comp, string_types): - states.add(comp) + states[comp] = True; top[saltenv][tgt] = matches - top[saltenv][tgt].extend(list(states)) + top[saltenv][tgt].extend(list(states.keys())) return self.sort_top_targets(top, orders) def sort_top_targets(self, top, orders): From ae1ad56bcd227ca9fd5c68418da334fc62bf2a8c Mon Sep 17 00:00:00 2001 From: Vahid Hamidullah Date: Mon, 16 Dec 2013 16:54:30 -0500 Subject: [PATCH 234/769] Remove trailing semicolons from previous commit --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 20f4ca7..289bccb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -253,7 +253,7 @@ def merge_tops(self, tops): continue for tgt in targets: matches = [] - states = OrderedDict(); + states = OrderedDict() orders[saltenv][tgt] = 0 for comp in ctop[saltenv][tgt]: if isinstance(comp, dict): @@ -268,7 +268,7 @@ def merge_tops(self, tops): order = 0 orders[saltenv][tgt] = order if isinstance(comp, string_types): - states[comp] = True; + states[comp] = True top[saltenv][tgt] = matches top[saltenv][tgt].extend(list(states.keys())) return self.sort_top_targets(top, orders) From da08a158c490e06f95cc77d524fa19103b9e5573 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 Jan 2014 14:16:03 -0700 Subject: [PATCH 235/769] Make the functions only get generated once instead of every pillar --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 961b228..266012e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -89,7 +89,7 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, saltenv, ext=None): + def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client @@ -98,7 +98,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None): if opts.get('file_client', '') == 'local': opts['grains'] = grains self.functions = salt.loader.minion_mods(opts) - else: + elif functions is None: self.functions = salt.loader.minion_mods(self.opts) self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) From ae65d24a289a55d4d21e5887133147a32bf44937 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 6 Jan 2014 13:30:38 -0800 Subject: [PATCH 236/769] Keep the functions if they are passed in --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 266012e..94fff6a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -100,6 +100,8 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.functions = salt.loader.minion_mods(opts) elif functions is None: self.functions = salt.loader.minion_mods(self.opts) + else: + self.functions = functions self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real From 49b4948ed4b1f99a4ddc7dcfe5dc6af96bb883d4 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 6 Jan 2014 13:34:57 -0800 Subject: [PATCH 237/769] Break out the function loading into a seperate if block --- src/saltext/consul/pillar/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 94fff6a..ff1b8d1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -95,13 +95,15 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # use the local file client self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) self.client = salt.fileclient.get_file_client(self.opts) - if opts.get('file_client', '') == 'local': - opts['grains'] = grains + # if we didn't pass in functions, lets load them + if functions is None: self.functions = salt.loader.minion_mods(opts) - elif functions is None: - self.functions = salt.loader.minion_mods(self.opts) else: self.functions = functions + + if opts.get('file_client', '') == 'local': + opts['grains'] = grains + self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real From f4de1a3bbf6caabc98ae02cbeb1ca1b761ea73c1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 Jan 2014 15:13:37 -0700 Subject: [PATCH 238/769] Attempt at fixing some of the pillar platform tests --- src/saltext/consul/pillar/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ff1b8d1..e93d1c1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -95,15 +95,19 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # use the local file client self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) self.client = salt.fileclient.get_file_client(self.opts) - # if we didn't pass in functions, lets load them - if functions is None: - self.functions = salt.loader.minion_mods(opts) - else: - self.functions = functions if opts.get('file_client', '') == 'local': opts['grains'] = grains + # if we didn't pass in functions, lets load them + elif functions is None: + if opts.get('file_client', '') == 'local': + self.functions = salt.loader.minion_mods(opts) + else: + self.functions = salt.loader.minion_mods(self.opts) + else: + self.functions = functions + self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real From 46bbacb2d488b6ffb4399f99c932b5cfd148918e Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 6 Jan 2014 17:17:05 -0700 Subject: [PATCH 239/769] Do! if, not elif --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e93d1c1..5891248 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -100,7 +100,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): opts['grains'] = grains # if we didn't pass in functions, lets load them - elif functions is None: + if functions is None: if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) else: From 36724d1a1b7ede9cb446d775ae0d6c9d8f0ba425 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 6 Jan 2014 16:31:53 -0800 Subject: [PATCH 240/769] Make sure we load ufnctions in ALL situations --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e93d1c1..5891248 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -100,7 +100,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): opts['grains'] = grains # if we didn't pass in functions, lets load them - elif functions is None: + if functions is None: if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) else: From 486fa130d1676cc4e558003b6a2794cddabf92b1 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Fri, 20 Dec 2013 16:51:42 -0700 Subject: [PATCH 241/769] Proxy minion --- src/saltext/consul/__init__.py | 103 +++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8985f5d..5a4d798 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -235,6 +235,109 @@ def shutdown(self): ''' +class ProxyMinion(parsers.MinionOptionParser): + ''' + Create a proxy minion server + ''' + def prepare(self): + ''' + Run the preparation sequence required to start a salt minion. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).prepare() + ''' + self.parse_args() + + try: + if self.config['verify_env']: + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) + confd = os.path.join( + os.path.dirname(self.config['conf_file']), confd + ) + else: + confd = os.path.join( + os.path.dirname(self.config['conf_file']), 'minion.d' + ) + verify_env( + [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ], + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], + ) + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): + # Logfile is not using Syslog, verify + verify_files([logfile], self.config['user']) + except OSError as err: + sys.exit(err.errno) + + self.setup_logfile_logger() + logger.info( + 'Setting up a Salt Proxy Minion "{0}"'.format( + self.config['id'] + ) + ) + migrations.migrate_paths(self.config) + # Late import so logging works correctly + import salt.minion + # If the minion key has not been accepted, then Salt enters a loop + # waiting for it, if we daemonize later then the minion could halt + # the boot process waiting for a key to be accepted on the master. + # This is the latest safe place to daemonize + self.daemonize_if_required() + self.set_pidfile() + if isinstance(self.config.get('master'), list): + self.minion = salt.minion.MultiMinion(self.config) + else: + self.minion = salt.minion.ProxyMinion(self.config) + + def start(self): + ''' + Start the actual minion. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).start() + + NOTE: Run any required code before calling `super()`. + ''' + self.prepare() + try: + if check_user(self.config['user']): + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + finally: + self.shutdown() + + def shutdown(self): + ''' + If sub-classed, run any shutdown operations on this method. + ''' + + class Syndic(parsers.SyndicOptionParser): ''' Create a syndic server From 5467dbcec1ee0be5400f9403e4a75197a551cee1 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Wed, 15 Jan 2014 13:00:59 -0700 Subject: [PATCH 242/769] Proxy minion support. --- src/saltext/consul/__init__.py | 39 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5a4d798..c278f84 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# coding: utf-8 -*- ''' Make me some salt! ''' @@ -239,7 +239,7 @@ class ProxyMinion(parsers.MinionOptionParser): ''' Create a proxy minion server ''' - def prepare(self): + def prepare(self, proxydetails): ''' Run the preparation sequence required to start a salt minion. @@ -280,7 +280,10 @@ def prepare(self): permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], ) - logfile = self.config['log_file'] + if 'proxy_log' in proxydetails: + logfile = proxydetails['proxy_log'] + else: + logfile = None if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): @@ -289,6 +292,7 @@ def prepare(self): except OSError as err: sys.exit(err.errno) + self.config['proxy'] = proxydetails self.setup_logfile_logger() logger.info( 'Setting up a Salt Proxy Minion "{0}"'.format( @@ -309,7 +313,7 @@ def prepare(self): else: self.minion = salt.minion.ProxyMinion(self.config) - def start(self): + def start(self, proxydetails): ''' Start the actual minion. @@ -319,23 +323,26 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - self.prepare() - try: - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - finally: - self.shutdown() + self.prepare(proxydetails) + self.minion.tune_in() + # try: + # if check_user(self.config['user']): + # except (KeyboardInterrupt, SaltSystemExit) as exc: + # logger.warn('Stopping the Salt Minion') + # if isinstance(exc, KeyboardInterrupt): + # logger.warn('Exiting on Ctrl-c') + # else: + # logger.error(str(exc)) + # finally: + # self.shutdown() def shutdown(self): ''' If sub-classed, run any shutdown operations on this method. ''' + if 'proxy' in self.minion.opts: + self.minion.opts['proxyobject'].shutdown(self.minion.opts) + class Syndic(parsers.SyndicOptionParser): From c94852e699095dd4749cae112053240a0e116a0e Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Wed, 15 Jan 2014 13:47:59 -0700 Subject: [PATCH 243/769] Fix some pylint errors, remove salt_proxy_minion function in scripts. --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c278f84..a043abb 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -344,7 +344,6 @@ def shutdown(self): self.minion.opts['proxyobject'].shutdown(self.minion.opts) - class Syndic(parsers.SyndicOptionParser): ''' Create a syndic server From 0e8d29ef05e2d9e8217e483268b4139d089627f4 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Thu, 16 Jan 2014 14:17:18 -0700 Subject: [PATCH 244/769] Fix stacktrace on ^C of proxy-minion. Refactor ProxyMinion class to inherit from Minion. --- src/saltext/consul/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a043abb..bc988ed 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -324,17 +324,16 @@ def start(self, proxydetails): NOTE: Run any required code before calling `super()`. ''' self.prepare(proxydetails) - self.minion.tune_in() - # try: - # if check_user(self.config['user']): - # except (KeyboardInterrupt, SaltSystemExit) as exc: - # logger.warn('Stopping the Salt Minion') - # if isinstance(exc, KeyboardInterrupt): - # logger.warn('Exiting on Ctrl-c') - # else: - # logger.error(str(exc)) - # finally: - # self.shutdown() + try: + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Proxy Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + finally: + self.shutdown() def shutdown(self): ''' From 44bbcf97a64e62c8965855140e055617cb138061 Mon Sep 17 00:00:00 2001 From: Amir Pakdel Date: Tue, 28 Jan 2014 11:03:47 -0500 Subject: [PATCH 245/769] Fixed issue #9932 If an ext_pillar item is not loaded and its branch is not present in pillar_roots, the corresponding salt environment will not be available. However, this is not an issue, since it will be available when all of ext_pillar items are parsed; therefore, when trying to get the SLS from that non-present salt environment, we should merely log a debug message and continue on. --- src/saltext/consul/pillar/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5891248..e5e0a76 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -341,10 +341,18 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: - msg = ('Specified SLS {0!r} in environment {1!r} is not' - ' available on the salt master').format(sls, saltenv) - log.error(msg) - errors.append(msg) + if self.opts['pillar_roots'].get(saltenv): + msg = ('Specified SLS {0!r} in environment {1!r} is not' + ' available on the salt master').format(sls, saltenv) + log.error(msg) + errors.append(msg) + else: + log.debug('Specified SLS {0!r} in environment {1!r} is not' + ' found, which might be due to environment {1!r}' + ' not being present in "pillar_roots" yet!' + .format(sls, saltenv)) + # return state, mods, errors + return None, mods, errors state = None try: state = compile_template( From f1d3f2746e46877714167aacf3dd585133eaa51e Mon Sep 17 00:00:00 2001 From: vs Date: Tue, 28 Jan 2014 18:04:41 +0100 Subject: [PATCH 246/769] added minion_denied to pki-structure to easily find denied minions --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index bc988ed..fe440c5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -70,6 +70,7 @@ def prepare(self): self.config['pki_dir'], os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), + os.path.join(self.config['pki_dir'], 'minions_denied'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], From 65aa65078196c5658509d84e23fad81654558ae7 Mon Sep 17 00:00:00 2001 From: vs Date: Wed, 29 Jan 2014 15:44:27 +0100 Subject: [PATCH 247/769] denied minions keys are written to $pki_dir/minions_denied --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index bc988ed..fe440c5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -70,6 +70,7 @@ def prepare(self): self.config['pki_dir'], os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), + os.path.join(self.config['pki_dir'], 'minions_denied'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], From c7033fc8f49c0aa9afa27bf715c1e752a95cb228 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 6 Feb 2014 12:27:51 -0700 Subject: [PATCH 248/769] Set the umask to 0077 when running verify on the minion log file to ensure that log file is created with 0600 if it does not already exist. --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fe440c5..a9ed74a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -183,7 +183,9 @@ def prepare(self): 'udp://', 'file://')): # Logfile is not using Syslog, verify + current_umask = os.umask(0077) verify_files([logfile], self.config['user']) + os.umask(current_umask) except OSError as err: sys.exit(err.errno) From 3976abfa4c5093a547482834afe82d63779233d5 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 24 Feb 2014 16:34:22 -0700 Subject: [PATCH 249/769] Add initial hook to load in the ioflo master and minion daemons --- src/saltext/consul/__init__.py | 51 ++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a9ed74a..fc67160 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -95,16 +95,21 @@ def prepare(self): self.setup_logfile_logger() logger.info('Setting up the Salt Master') - if not verify_socket(self.config['interface'], - self.config['publish_port'], - self.config['ret_port']): - self.exit(4, 'The ports are not available to bind\n') - self.config['interface'] = ip_bracket(self.config['interface']) - migrations.migrate_paths(self.config) - - # Late import so logging works correctly - import salt.master - self.master = salt.master.Master(self.config) + if self.config['transport'].lower() == 'zeromq': + if not verify_socket(self.config['interface'], + self.config['publish_port'], + self.config['ret_port']): + self.exit(4, 'The ports are not available to bind\n') + self.config['interface'] = ip_bracket(self.config['interface']) + migrations.migrate_paths(self.config) + + # Late import so logging works correctly + import salt.master + self.master = salt.master.Master(self.config) + else: + # Add a udp port check here + import salt.daemons + self.master = salt.daemons.IoFloMaster(self.config) self.daemonize_if_required() self.set_pidfile() @@ -196,18 +201,22 @@ def prepare(self): ) ) migrations.migrate_paths(self.config) - # Late import so logging works correctly - import salt.minion - # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion could halt - # the boot process waiting for a key to be accepted on the master. - # This is the latest safe place to daemonize - self.daemonize_if_required() - self.set_pidfile() - if isinstance(self.config.get('master'), list): - self.minion = salt.minion.MultiMinion(self.config) + if self.config['transport'].lower() == 'zeromq': + # Late import so logging works correctly + import salt.minion + # If the minion key has not been accepted, then Salt enters a loop + # waiting for it, if we daemonize later then the minion could halt + # the boot process waiting for a key to be accepted on the master. + # This is the latest safe place to daemonize + self.daemonize_if_required() + self.set_pidfile() + if isinstance(self.config.get('master'), list): + self.minion = salt.minion.MultiMinion(self.config) + else: + self.minion = salt.minion.Minion(self.config) else: - self.minion = salt.minion.Minion(self.config) + import salt.daemons + self.minion = salt.daemons.IoFloMinion(self.config) def start(self): ''' From 418695e5d756441ea1af7feaaa80015f2a700083 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 11:01:17 -0700 Subject: [PATCH 250/769] move ioflo init classes into salt/daemons/flo --- src/saltext/consul/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fc67160..8ef5e60 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -108,8 +108,8 @@ def prepare(self): self.master = salt.master.Master(self.config) else: # Add a udp port check here - import salt.daemons - self.master = salt.daemons.IoFloMaster(self.config) + import salt.daemons.flo + self.master = salt.daemons.flo.IoFloMaster(self.config) self.daemonize_if_required() self.set_pidfile() @@ -215,8 +215,8 @@ def prepare(self): else: self.minion = salt.minion.Minion(self.config) else: - import salt.daemons - self.minion = salt.daemons.IoFloMinion(self.config) + import salt.daemons.flo + self.minion = salt.daemons.flo.IoFloMinion(self.config) def start(self): ''' From eb5c29b229f26db523911eb4117120316e2ec1d4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 14:32:45 -0700 Subject: [PATCH 251/769] Call the corect ioflo gateway class --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8ef5e60..69eff73 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -109,7 +109,7 @@ def prepare(self): else: # Add a udp port check here import salt.daemons.flo - self.master = salt.daemons.flo.IoFloMaster(self.config) + self.master = salt.daemons.flo.IofloMaster(self.config) self.daemonize_if_required() self.set_pidfile() @@ -216,7 +216,7 @@ def prepare(self): self.minion = salt.minion.Minion(self.config) else: import salt.daemons.flo - self.minion = salt.daemons.flo.IoFloMinion(self.config) + self.minion = salt.daemons.flo.IofloMinion(self.config) def start(self): ''' From 2384865802ac02928ff4116a83cbaca836543218 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 25 Feb 2014 15:59:46 -0700 Subject: [PATCH 252/769] Create raet key dirs --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 69eff73..b443e1e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -71,6 +71,9 @@ def prepare(self): os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), os.path.join(self.config['pki_dir'], 'minions_denied'), + os.path.join(self.config['pki_dir'], 'accepted'), + os.path.join(self.config['pki_dir'], 'pending'), + os.path.join(self.config['pki_dir'], 'rejected'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], From d97532f66b60daa71451adbe866601ddf152b9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Pitucha?= Date: Sun, 9 Mar 2014 02:21:14 +0000 Subject: [PATCH 253/769] Report errors when preparation fails Previously salt daemons could fail to start without giving any reason, when (for example) chown failed during the startup. This change doesn't fix anything, but at least reports the error back to the user. --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b443e1e..9a6caeb 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -93,6 +93,7 @@ def prepare(self): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: + logger.exception('Failed to prepare salt environment') sys.exit(err.errno) self.setup_logfile_logger() @@ -195,6 +196,7 @@ def prepare(self): verify_files([logfile], self.config['user']) os.umask(current_umask) except OSError as err: + logger.exception('Failed to prepare salt environment') sys.exit(err.errno) self.setup_logfile_logger() @@ -305,6 +307,7 @@ def prepare(self, proxydetails): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: + logger.exception('Failed to prepare salt environment') sys.exit(err.errno) self.config['proxy'] = proxydetails @@ -392,6 +395,7 @@ def prepare(self): # Logfile is not using Syslog, verify verify_files([logfile], self.config['user']) except OSError as err: + logger.exception('Failed to prepare salt environment') sys.exit(err.errno) self.setup_logfile_logger() From 7d4d09b2a9165683402d120b236af30bcc2aca43 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 11 Mar 2014 14:45:05 -0600 Subject: [PATCH 254/769] Prevent endless recursion from external pillars --- src/saltext/consul/pillar/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e5e0a76..0659a16 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -488,14 +488,15 @@ def ext_pillar(self, pillar): ) return pillar - def compile_pillar(self): + def compile_pillar(self, ext=True): ''' Render the pillar data and return ''' top, terrors = self.get_top() matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - self.ext_pillar(pillar) + if ext: + self.ext_pillar(pillar) errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) From 2ccacba8747c388922c7257883a800fede1cb66a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 13 Mar 2014 00:00:03 -0600 Subject: [PATCH 255/769] Extend verify_env to cover raet key dirs if raet is enabled --- src/saltext/consul/__init__.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9a6caeb..9e3d4c1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -65,15 +65,11 @@ def prepare(self): try: if self.config['verify_env']: - verify_env( - [ + v_dirs = [ self.config['pki_dir'], os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), os.path.join(self.config['pki_dir'], 'minions_denied'), - os.path.join(self.config['pki_dir'], 'accepted'), - os.path.join(self.config['pki_dir'], 'pending'), - os.path.join(self.config['pki_dir'], 'rejected'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], @@ -81,7 +77,13 @@ def prepare(self): os.path.join(self.config['cachedir'], 'proc'), self.config['sock_dir'], self.config['token_dir'], - ], + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + verify_env( + v_dirs, self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], @@ -175,14 +177,19 @@ def prepare(self): confd = os.path.join( os.path.dirname(self.config['conf_file']), 'minion.d' ) - verify_env( - [ + v_dirs = [ self.config['pki_dir'], self.config['cachedir'], self.config['sock_dir'], self.config['extension_modules'], confd, - ], + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + verify_env( + v_dirs, self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], From 23a18f0a8a171f0b7d8bed4f70b98607d1bde7ed Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 27 Mar 2014 16:36:05 -0600 Subject: [PATCH 256/769] Fix broken pillar and broken grains context inside rendered pillars. Closes #11453 --- src/saltext/consul/pillar/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0659a16..74b2939 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -95,7 +95,6 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # use the local file client self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) self.client = salt.fileclient.get_file_client(self.opts) - if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -103,10 +102,8 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): if functions is None: if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) - else: - self.functions = salt.loader.minion_mods(self.opts) else: - self.functions = functions + self.functions = salt.loader.minion_mods(self.opts) self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) From 5ea8fc9810fb4d773183f7cb442328d44490a81d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 31 Mar 2014 02:33:18 +0100 Subject: [PATCH 257/769] Revert "Fix broken pillar and broken grains context inside rendered pillars." This reverts commit 23a18f0a8a171f0b7d8bed4f70b98607d1bde7ed. --- src/saltext/consul/pillar/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 74b2939..0659a16 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -95,6 +95,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # use the local file client self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) self.client = salt.fileclient.get_file_client(self.opts) + if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -102,8 +103,10 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): if functions is None: if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts) + else: + self.functions = salt.loader.minion_mods(self.opts) else: - self.functions = salt.loader.minion_mods(self.opts) + self.functions = functions self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) From fdc8947eb8c13869cc6f74482d2b952935b4b482 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 4 Apr 2014 14:11:27 -0600 Subject: [PATCH 258/769] Modify funcs with grains in pillar. --- src/saltext/consul/pillar/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0659a16..0cc838b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -7,6 +7,7 @@ import os import collections import logging +import copy # Import salt libs import salt.loader From fc29b4ee15d75c96c7206227aa7aaafd55375ebc Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 8 Apr 2014 15:21:49 -0600 Subject: [PATCH 259/769] Catch rendering issues. Catch and warn on rendering issues in top files. Closes #8799. --- src/saltext/consul/pillar/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0cc838b..6002a61 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -304,7 +304,12 @@ def get_top(self): Returns the high data derived from the top file ''' tops, errors = self.get_tops() - return self.merge_tops(tops), errors + try: + merged_tops = self.merge_tops(tops) + except TypeError as err: + merged_tops = OrderedDict() + errors.append('Error encountered while render pillar top file.') + return merged_tops, errors def top_matches(self, top): ''' From 586ba2fe2e9101ca412f9699d25876ef59cbc962 Mon Sep 17 00:00:00 2001 From: Xavier Barbosa Date: Wed, 9 Apr 2014 06:39:46 +0200 Subject: [PATCH 260/769] implement merging strategy into pillar --- src/saltext/consul/pillar/__init__.py | 60 ++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0cc838b..6bb52ac 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -7,7 +7,7 @@ import os import collections import logging -import copy +from copy import copy # Import salt libs import salt.loader @@ -18,6 +18,7 @@ from salt._compat import string_types from salt.template import compile_template from salt.utils.dictupdate import update +from salt.utils.serializers.sls import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ @@ -25,6 +26,15 @@ log = logging.getLogger(__name__) +def merge_recurse(obj_a, obj_b): + copied = copy(obj_a) + return update(copied, obj_b) + + +def merge_aggregate(obj_a, obj_b): + return merge_recursive(obj_a, obj_b, level=1) + + def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): ''' Return the correct pillar driver based on the file_client option @@ -115,6 +125,11 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots + + self.merge_strategy = 'smart' + if opts.get('pillar_source_merging_strategy'): + self.merge_strategy = opts['pillar_source_merging_strategy'] + self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) def __valid_ext(self, ext): @@ -395,9 +410,12 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): ) if nstate: if key: - state[key] = nstate - else: - state.update(nstate) + nstate = { + key: nstate + } + + state = self.merge_sources(state, nstate) + if err: errors += err return state, mods, errors @@ -429,7 +447,7 @@ def render_pillar(self, matches): ) ) continue - update(pillar, pstate) + pillar = self.merge_sources(pillar, pstate) return pillar, errors @@ -438,10 +456,10 @@ def ext_pillar(self, pillar): Render the external pillar data ''' if not 'ext_pillar' in self.opts: - return {} + return pillar if not isinstance(self.opts['ext_pillar'], list): log.critical('The "ext_pillar" option is malformed') - return {} + return pillar for run in self.opts['ext_pillar']: if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') @@ -462,7 +480,7 @@ def ext_pillar(self, pillar): ext = self.ext_pillars[key](self.opts['id'], pillar, *val) else: ext = self.ext_pillars[key](self.opts['id'], pillar, val) - update(pillar, ext) + pillar = self.merge_sources(pillar, ext) except TypeError as e: if e.message.startswith('ext_pillar() takes exactly '): @@ -478,7 +496,7 @@ def ext_pillar(self, pillar): ext = self.ext_pillars[key](pillar, *val) else: ext = self.ext_pillars[key](pillar, val) - update(pillar, ext) + pillar = self.merge_sources(pillar, ext) except Exception as exc: log.exception( @@ -489,6 +507,28 @@ def ext_pillar(self, pillar): ) return pillar + def merge_sources(self, obj_a, obj_b): + strategy = self.merge_strategy + + if strategy == 'smart': + renderer = self.opts.get('renderer', 'yaml') + if renderer == 'sls' or renderer.startswith('sls_'): + strategy = 'aggregate' + else: + strategy = 'recurse' + + if strategy == 'recurse': + merged = merge_recurse(obj_a, obj_b) + elif strategy == 'aggregate': + #: level = 1 merge at least root data + merged = merge_aggregate(obj_a, obj_b) + else: + log.warning('unknown merging strategy {}, ' + 'fallback to recurse'.format(strategy)) + merged = merge_recurse(obj_a, obj_b) + + return merged + def compile_pillar(self, ext=True): ''' Render the pillar data and return @@ -497,7 +537,7 @@ def compile_pillar(self, ext=True): matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) if ext: - self.ext_pillar(pillar) + pillar = self.ext_pillar(pillar) errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) From f736e7eb749bd8f87e7e26535d637986abddeb56 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 14 Apr 2014 15:37:47 -0600 Subject: [PATCH 261/769] Fix External Raet REfactoring bugs --- src/saltext/consul/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9e3d4c1..74f0fb1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -131,12 +131,13 @@ def start(self): ''' self.prepare() if check_user(self.config['user']): - try: - self.master.start() - except MasterExit: - self.shutdown() - finally: - sys.exit() + self.master.start() + #try: + #self.master.start() + #except MasterExit: + #self.shutdown() + #finally: + #sys.exit() def shutdown(self): ''' From 74356b71b2f29f7257498248f1e5feb1bace1b8f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 15 Apr 2014 11:41:08 -0500 Subject: [PATCH 262/769] Remove unused import --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6002a61..5e6c32b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -7,7 +7,6 @@ import os import collections import logging -import copy # Import salt libs import salt.loader From 435bfa162b49a771cbeac70b55f5cacd3bb6efc4 Mon Sep 17 00:00:00 2001 From: s8weber Date: Wed, 23 Apr 2014 19:01:32 -0400 Subject: [PATCH 263/769] Minion restart if master is not responding --- src/saltext/consul/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 74f0fb1..c1c27e8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,6 +7,8 @@ import os import sys import warnings +import time +from random import randint # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( @@ -41,7 +43,7 @@ except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise -from salt.exceptions import SaltSystemExit, MasterExit +from salt.exceptions import SaltSystemExit, MasterExit, SaltClientError # Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint @@ -241,8 +243,8 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - self.prepare() try: + self.prepare() if check_user(self.config['user']): self.minion.tune_in() except (KeyboardInterrupt, SaltSystemExit) as exc: @@ -251,6 +253,14 @@ def start(self): logger.warn('Exiting on Ctrl-c') else: logger.error(str(exc)) + except SaltClientError as exc: + logger.error(exc) + if self.config.get('restart_on_error'): + logger.warn('** Restarting minion **') + s = randint(0, self.config.get('random_reauth_delay',10)) + logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) + time.sleep(s) + return 'reconnect' finally: self.shutdown() From ce1e17846c17b0064b3418ba8c76bfdf7620190d Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 23 Apr 2014 14:23:39 -0600 Subject: [PATCH 264/769] Use salt outputter, default queues directory --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 74f0fb1..69d219b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -77,6 +77,7 @@ def prepare(self): os.path.join(self.config['cachedir'], 'proc'), self.config['sock_dir'], self.config['token_dir'], + self.config['queue_dir'], ] if self.config.get('transport') == 'raet': v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) From 89d8018fb75a1843123940a4466ef8a4c7203c32 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 23 Apr 2014 18:00:24 -0600 Subject: [PATCH 265/769] update verify_env to use proper sqlite dir --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 69d219b..13634a5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -77,7 +77,7 @@ def prepare(self): os.path.join(self.config['cachedir'], 'proc'), self.config['sock_dir'], self.config['token_dir'], - self.config['queue_dir'], + self.config['sqlite_queue_dir'], ] if self.config.get('transport') == 'raet': v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) From bcc1c08c4fec3efb9cebf6b0223a41bf5a2eb2da Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 30 Apr 2014 15:08:13 -0600 Subject: [PATCH 266/769] add missing comma after ',' --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e441d61..b021ea4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -258,7 +258,7 @@ def start(self): logger.error(exc) if self.config.get('restart_on_error'): logger.warn('** Restarting minion **') - s = randint(0, self.config.get('random_reauth_delay',10)) + s = randint(0, self.config.get('random_reauth_delay', 10)) logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) time.sleep(s) return 'reconnect' From 74dfe7d18aed325a49281f97ab056f47057f487f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 30 Apr 2014 11:39:05 +0100 Subject: [PATCH 267/769] Use proper variable name --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5e6c32b..6707a84 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -468,8 +468,8 @@ def ext_pillar(self, pillar): ext = self.ext_pillars[key](self.opts['id'], pillar, val) update(pillar, ext) - except TypeError as e: - if e.message.startswith('ext_pillar() takes exactly '): + except TypeError as exc: + if exc.message.startswith('ext_pillar() takes exactly '): log.warning('Deprecation warning: ext_pillar "{0}"' ' needs to accept minion_id as first' ' argument'.format(key)) From 29936e1686585787ec5266462a959f656564ec49 Mon Sep 17 00:00:00 2001 From: steverweber Date: Sat, 10 May 2014 13:46:03 -0400 Subject: [PATCH 268/769] auto accept keys in minions_autosign Accept keyids if a file with the same name is found in minions_autosign. Also option to check the datestamp of the file and only accept if within timeout. --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b021ea4..89d698c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -72,6 +72,8 @@ def prepare(self): os.path.join(self.config['pki_dir'], 'minions'), os.path.join(self.config['pki_dir'], 'minions_pre'), os.path.join(self.config['pki_dir'], 'minions_denied'), + os.path.join(self.config['pki_dir'], + 'minions_autosign'), os.path.join(self.config['pki_dir'], 'minions_rejected'), self.config['cachedir'], From 1ecfa1a0e81a1bf8a78036712ab4a2ca44d97252 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 29 May 2014 11:46:13 +0100 Subject: [PATCH 269/769] PyLint inline ignore W0640(cell-var-from-loop) --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6707a84..091f380 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -291,11 +291,13 @@ def sort_top_targets(self, top, orders): Returns the sorted high data from the merged top files ''' sorted_top = collections.defaultdict(OrderedDict) + # pylint: disable=cell-var-from-loop for saltenv, targets in top.items(): sorted_targets = sorted(targets.keys(), key=lambda target: orders[saltenv][target]) for target in sorted_targets: sorted_top[saltenv][target] = targets[target] + # pylint: enable=cell-var-from-loop return sorted_top def get_top(self): From b866599f3c9e2c128a79cc93a9fb27dcc58639be Mon Sep 17 00:00:00 2001 From: vs Date: Fri, 30 May 2014 05:39:37 -0700 Subject: [PATCH 270/769] if master if of type list, check if master_type is set, else start MultiMinion --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 89d698c..439742c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -229,7 +229,10 @@ def prepare(self): self.daemonize_if_required() self.set_pidfile() if isinstance(self.config.get('master'), list): - self.minion = salt.minion.MultiMinion(self.config) + if self.config.get('master_type') == 'failover': + self.minion = salt.minion.Minion(self.config) + else: + self.minion = salt.minion.MultiMinion(self.config) else: self.minion = salt.minion.Minion(self.config) else: From d092c3202fe9c75096a5e7c6e69f4038ae925ff4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 30 May 2014 09:28:20 -0600 Subject: [PATCH 271/769] rename the sls renderer to the yamlex renderer --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 9c1c7e8..bc8e971 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -18,7 +18,7 @@ from salt._compat import string_types from salt.template import compile_template from salt.utils.dictupdate import update -from salt.utils.serializers.sls import merge_recursive +from salt.utils.serializers.yamlex import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ @@ -519,7 +519,7 @@ def merge_sources(self, obj_a, obj_b): if strategy == 'smart': renderer = self.opts.get('renderer', 'yaml') - if renderer == 'sls' or renderer.startswith('sls_'): + if renderer == 'yamlex' or renderer.startswith('yamlex_'): strategy = 'aggregate' else: strategy = 'recurse' From d4d6dda29d022efde4acb977a49b33b32edad972 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 30 May 2014 09:45:46 -0600 Subject: [PATCH 272/769] lint --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bc8e971..8735ed6 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -530,7 +530,7 @@ def merge_sources(self, obj_a, obj_b): #: level = 1 merge at least root data merged = merge_aggregate(obj_a, obj_b) else: - log.warning('unknown merging strategy {}, ' + log.warning('unknown merging strategy {0}, ' 'fallback to recurse'.format(strategy)) merged = merge_recurse(obj_a, obj_b) From f55bc0cc9fe8eec27f944132ec6f01d1261134c5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 6 Jun 2014 10:23:01 +0100 Subject: [PATCH 273/769] Fix PEP8 E713 - test for membership should be "not in" --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8735ed6..e58f036 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -462,7 +462,7 @@ def ext_pillar(self, pillar): ''' Render the external pillar data ''' - if not 'ext_pillar' in self.opts: + if 'ext_pillar' not in self.opts: return pillar if not isinstance(self.opts['ext_pillar'], list): log.critical('The "ext_pillar" option is malformed') From ee1dee4e3e77adf85721aaf5e290a216e4bb4579 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 13 Jun 2014 09:12:54 -0600 Subject: [PATCH 274/769] Rename db to sdb, to avoid name conflicts --- src/saltext/consul/sdb/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/saltext/consul/sdb/__init__.py diff --git a/src/saltext/consul/sdb/__init__.py b/src/saltext/consul/sdb/__init__.py new file mode 100644 index 0000000..e7c493a --- /dev/null +++ b/src/saltext/consul/sdb/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +''' +DB Module Directory +''' From 888c353064bcf47030d0515f0a87269b4f73d6ed Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Sun, 15 Jun 2014 08:54:40 -0600 Subject: [PATCH 275/769] Rename init --- src/saltext/consul/sdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/sdb/__init__.py b/src/saltext/consul/sdb/__init__.py index e7c493a..df17b62 100644 --- a/src/saltext/consul/sdb/__init__.py +++ b/src/saltext/consul/sdb/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- ''' -DB Module Directory +SDB Module Directory ''' From c38faf0e5c8119c796ba05c0d0dce5847868ec64 Mon Sep 17 00:00:00 2001 From: Steve Weber Date: Thu, 19 Jun 2014 13:41:39 -0400 Subject: [PATCH 276/769] move restart code to salt.Minion().Start() --- src/saltext/consul/__init__.py | 147 +++++++++++++++++---------------- 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 439742c..0ad70c2 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -162,63 +162,65 @@ def prepare(self): super(YourSubClass, self).prepare() ''' - self.parse_args() + + if not hasattr(self, 'config'): + self.parse_args() - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) + try: + if self.config['verify_env']: + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) + confd = os.path.join( + os.path.dirname(self.config['conf_file']), confd + ) + else: confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd + os.path.dirname(self.config['conf_file']), 'minion.d' ) - else: - confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' + v_dirs = [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + verify_env( + v_dirs, + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) - v_dirs = [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): + # Logfile is not using Syslog, verify + current_umask = os.umask(0077) + verify_files([logfile], self.config['user']) + os.umask(current_umask) + except OSError as err: + logger.exception('Failed to prepare salt environment') + sys.exit(err.errno) + + self.setup_logfile_logger() + logger.info( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0077) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] ) - ) - migrations.migrate_paths(self.config) + migrations.migrate_paths(self.config) if self.config['transport'].lower() == 'zeromq': # Late import so logging works correctly import salt.minion @@ -249,26 +251,29 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - try: - self.prepare() - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - except SaltClientError as exc: - logger.error(exc) - if self.config.get('restart_on_error'): - logger.warn('** Restarting minion **') - s = randint(0, self.config.get('random_reauth_delay', 10)) - logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) - time.sleep(s) - return 'reconnect' - finally: - self.shutdown() + reconnect = True + while reconnect: + reconnect = False + try: + self.prepare() + if check_user(self.config['user']): + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + except SaltClientError as exc: + logger.error(exc) + if self.config.get('restart_on_error'): + logger.warn('** Restarting minion **') + s = randint(0, self.config.get('random_reauth_delay', 10)) + logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) + time.sleep(s) + reconnect = True + finally: + self.shutdown() def shutdown(self): ''' From abd0df3b1ac31be91226cf0af0b8ae37a6e18384 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 20 Jun 2014 09:54:45 -0600 Subject: [PATCH 277/769] Lint per #13582 --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0ad70c2..4d8e5d8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -162,7 +162,6 @@ def prepare(self): super(YourSubClass, self).prepare() ''' - if not hasattr(self, 'config'): self.parse_args() From ac53d30da4b14ba925912e9fcc8cc037d7a51a71 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 7 Jul 2014 15:52:43 -0600 Subject: [PATCH 278/769] Ensure that the raet cachedir is created in the verify_env --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4d8e5d8..0c6f1a9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -87,6 +87,7 @@ def prepare(self): v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) verify_env( v_dirs, self.config['user'], @@ -195,6 +196,7 @@ def prepare(self): v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) verify_env( v_dirs, self.config['user'], From 3beceeb72f99b99ddbd9d2e5abf9f5224eb571af Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 7 Jul 2014 22:03:42 -0600 Subject: [PATCH 279/769] fix minion daemonize for raet --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0c6f1a9..fdd2f61 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -240,6 +240,8 @@ def prepare(self): self.minion = salt.minion.Minion(self.config) else: import salt.daemons.flo + self.daemonize_if_required() + self.set_pidfile() self.minion = salt.daemons.flo.IofloMinion(self.config) def start(self): From 264c20cfe7223c88259e8c408e8f8f405dcedabb Mon Sep 17 00:00:00 2001 From: Aditya Kulkarni Date: Tue, 1 Jul 2014 01:06:18 -0400 Subject: [PATCH 280/769] Make sure multiple git pillar sources work --- src/saltext/consul/pillar/__init__.py | 67 ++++++++++++++++++--------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e58f036..5348889 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -125,7 +125,6 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots - self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -458,7 +457,37 @@ def render_pillar(self, matches): return pillar, errors - def ext_pillar(self, pillar): + def _external_pillar_data(self, + pillar, + val, + pillar_dirs, + key): + ''' + Builds actual pillar data structure + and update + the variable ``pillar`` + ''' + + ext = None + + # try the new interface, which includes the minion ID + # as first argument + if isinstance(val, dict): + ext = self.ext_pillars[key](self.opts['id'], pillar, **val) + elif isinstance(val, list): + ext = self.ext_pillars[key](self.opts['id'], pillar, *val) + else: + if key == 'git': + ext = self.ext_pillars[key](self.opts['id'], + val, + pillar_dirs) + else: + ext = self.ext_pillars[key](self.opts['id'], + pillar, + val) + return ext + + def ext_pillar(self, pillar, pillar_dirs): ''' Render the external pillar data ''' @@ -467,6 +496,7 @@ def ext_pillar(self, pillar): if not isinstance(self.opts['ext_pillar'], list): log.critical('The "ext_pillar" option is malformed') return pillar + ext = None for run in self.opts['ext_pillar']: if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') @@ -479,16 +509,10 @@ def ext_pillar(self, pillar): continue try: try: - # try the new interface, which includes the minion ID - # as first argument - if isinstance(val, dict): - ext = self.ext_pillars[key](self.opts['id'], pillar, **val) - elif isinstance(val, list): - ext = self.ext_pillars[key](self.opts['id'], pillar, *val) - else: - ext = self.ext_pillars[key](self.opts['id'], pillar, val) - pillar = self.merge_sources(pillar, ext) - + ext = self._external_pillar_data(pillar, + val, + pillar_dirs, + key) except TypeError as exc: if exc.message.startswith('ext_pillar() takes exactly '): log.warning('Deprecation warning: ext_pillar "{0}"' @@ -497,14 +521,10 @@ def ext_pillar(self, pillar): else: raise - if isinstance(val, dict): - ext = self.ext_pillars[key](pillar, **val) - elif isinstance(val, list): - ext = self.ext_pillars[key](pillar, *val) - else: - ext = self.ext_pillars[key](pillar, val) - pillar = self.merge_sources(pillar, ext) - + ext = self._external_pillar_data(pillar, + val, + pillar_dirs, + key) except Exception as exc: log.exception( 'Failed to load ext_pillar {0}: {1}'.format( @@ -512,6 +532,9 @@ def ext_pillar(self, pillar): exc ) ) + if ext: + pillar = self.merge_sources(pillar, ext) + ext = None return pillar def merge_sources(self, obj_a, obj_b): @@ -536,7 +559,7 @@ def merge_sources(self, obj_a, obj_b): return merged - def compile_pillar(self, ext=True): + def compile_pillar(self, ext=True, pillar_dirs=None): ''' Render the pillar data and return ''' @@ -544,7 +567,7 @@ def compile_pillar(self, ext=True): matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) if ext: - pillar = self.ext_pillar(pillar) + pillar = self.ext_pillar(pillar, pillar_dirs) errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) From 0042af4bdb15dc2f4b382db7a28889573b54f19d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 14 Jul 2014 12:01:39 -0600 Subject: [PATCH 281/769] Remove stale code --- src/saltext/consul/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index fdd2f61..e994a28 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -138,12 +138,6 @@ def start(self): self.prepare() if check_user(self.config['user']): self.master.start() - #try: - #self.master.start() - #except MasterExit: - #self.shutdown() - #finally: - #sys.exit() def shutdown(self): ''' From 4c80d07c11e77ea1c5ed96d687a6c3f437490a07 Mon Sep 17 00:00:00 2001 From: steverweber Date: Wed, 9 Jul 2014 14:53:34 -0400 Subject: [PATCH 282/769] change minion auto restart logic into multiprocessing to fix mem leaks --- src/saltext/consul/__init__.py | 144 +++++++++++++++------------------ 1 file changed, 65 insertions(+), 79 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e994a28..b7b7287 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,8 +7,6 @@ import os import sys import warnings -import time -from random import randint # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( @@ -43,7 +41,7 @@ except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise -from salt.exceptions import SaltSystemExit, MasterExit, SaltClientError +from salt.exceptions import SaltSystemExit, MasterExit # Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint @@ -157,65 +155,64 @@ def prepare(self): super(YourSubClass, self).prepare() ''' - if not hasattr(self, 'config'): - self.parse_args() + self.parse_args() - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) - confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd - ) - else: + try: + if self.config['verify_env']: + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' + os.path.dirname(self.config['conf_file']), confd ) - v_dirs = [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + else: + confd = os.path.join( + os.path.dirname(self.config['conf_file']), 'minion.d' ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0077) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] + v_dirs = [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) + verify_env( + v_dirs, + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): + # Logfile is not using Syslog, verify + current_umask = os.umask(0077) + verify_files([logfile], self.config['user']) + os.umask(current_umask) + except OSError as err: + logger.exception('Failed to prepare salt environment') + sys.exit(err.errno) + + self.setup_logfile_logger() + logger.info( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] ) - migrations.migrate_paths(self.config) + ) + migrations.migrate_paths(self.config) if self.config['transport'].lower() == 'zeromq': # Late import so logging works correctly import salt.minion @@ -248,29 +245,18 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - reconnect = True - while reconnect: - reconnect = False - try: - self.prepare() - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - except SaltClientError as exc: - logger.error(exc) - if self.config.get('restart_on_error'): - logger.warn('** Restarting minion **') - s = randint(0, self.config.get('random_reauth_delay', 10)) - logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) - time.sleep(s) - reconnect = True - finally: - self.shutdown() + try: + self.prepare() + if check_user(self.config['user']): + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + finally: + self.shutdown() def shutdown(self): ''' From 0826d79890a3da9c38f16590532f3631cbef2e27 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 23 Jul 2014 13:46:23 -0600 Subject: [PATCH 283/769] Revert "change minion auto restart to multiprocessing mode" --- src/saltext/consul/__init__.py | 144 ++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 65 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b7b7287..e994a28 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,6 +7,8 @@ import os import sys import warnings +import time +from random import randint # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( @@ -41,7 +43,7 @@ except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise -from salt.exceptions import SaltSystemExit, MasterExit +from salt.exceptions import SaltSystemExit, MasterExit, SaltClientError # Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint @@ -155,64 +157,65 @@ def prepare(self): super(YourSubClass, self).prepare() ''' - self.parse_args() + if not hasattr(self, 'config'): + self.parse_args() - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) + try: + if self.config['verify_env']: + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) + confd = os.path.join( + os.path.dirname(self.config['conf_file']), confd + ) + else: confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd + os.path.dirname(self.config['conf_file']), 'minion.d' ) - else: - confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' + v_dirs = [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) + verify_env( + v_dirs, + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) - v_dirs = [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): + # Logfile is not using Syslog, verify + current_umask = os.umask(0077) + verify_files([logfile], self.config['user']) + os.umask(current_umask) + except OSError as err: + logger.exception('Failed to prepare salt environment') + sys.exit(err.errno) + + self.setup_logfile_logger() + logger.info( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0077) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] ) - ) - migrations.migrate_paths(self.config) + migrations.migrate_paths(self.config) if self.config['transport'].lower() == 'zeromq': # Late import so logging works correctly import salt.minion @@ -245,18 +248,29 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - try: - self.prepare() - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - finally: - self.shutdown() + reconnect = True + while reconnect: + reconnect = False + try: + self.prepare() + if check_user(self.config['user']): + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + except SaltClientError as exc: + logger.error(exc) + if self.config.get('restart_on_error'): + logger.warn('** Restarting minion **') + s = randint(0, self.config.get('random_reauth_delay', 10)) + logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) + time.sleep(s) + reconnect = True + finally: + self.shutdown() def shutdown(self): ''' From a78db612ceb55fce4d173481f1eabe19bb432fa4 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 23 Jul 2014 15:31:24 -0600 Subject: [PATCH 284/769] Fix pillar overlay bug --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5348889..8e366d1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -272,8 +272,8 @@ def merge_tops(self, tops): ''' Cleanly merge the top files ''' - top = collections.defaultdict(dict) - orders = collections.defaultdict(dict) + top = collections.defaultdict(OrderedDict) + orders = collections.defaultdict(OrderedDict) for ctops in tops.values(): for ctop in ctops: for saltenv, targets in ctop.items(): From 4fb7d14952dacfe7c91fe9b01772b9fc64f559ce Mon Sep 17 00:00:00 2001 From: steverweber Date: Thu, 24 Jul 2014 12:25:57 -0400 Subject: [PATCH 285/769] fix issues with keepalive minion --- src/saltext/consul/__init__.py | 144 +++++++++++++++------------------ 1 file changed, 65 insertions(+), 79 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e994a28..b7b7287 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,8 +7,6 @@ import os import sys import warnings -import time -from random import randint # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( @@ -43,7 +41,7 @@ except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise -from salt.exceptions import SaltSystemExit, MasterExit, SaltClientError +from salt.exceptions import SaltSystemExit, MasterExit # Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint @@ -157,65 +155,64 @@ def prepare(self): super(YourSubClass, self).prepare() ''' - if not hasattr(self, 'config'): - self.parse_args() + self.parse_args() - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) - confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd - ) - else: + try: + if self.config['verify_env']: + confd = self.config.get('default_include') + if confd: + # If 'default_include' is specified in config, then use it + if '*' in confd: + # Value is of the form "minion.d/*.conf" + confd = os.path.dirname(confd) + if not os.path.isabs(confd): + # If configured 'default_include' is not an absolute + # path, consider it relative to folder of 'conf_file' + # (/etc/salt by default) confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' + os.path.dirname(self.config['conf_file']), confd ) - v_dirs = [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], + else: + confd = os.path.join( + os.path.dirname(self.config['conf_file']), 'minion.d' ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0077) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] + v_dirs = [ + self.config['pki_dir'], + self.config['cachedir'], + self.config['sock_dir'], + self.config['extension_modules'], + confd, + ] + if self.config.get('transport') == 'raet': + v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) + v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) + v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) + verify_env( + v_dirs, + self.config['user'], + permissive=self.config['permissive_pki_access'], + pki_dir=self.config['pki_dir'], ) + logfile = self.config['log_file'] + if logfile is not None and not logfile.startswith(('tcp://', + 'udp://', + 'file://')): + # Logfile is not using Syslog, verify + current_umask = os.umask(0077) + verify_files([logfile], self.config['user']) + os.umask(current_umask) + except OSError as err: + logger.exception('Failed to prepare salt environment') + sys.exit(err.errno) + + self.setup_logfile_logger() + logger.info( + 'Setting up the Salt Minion "{0}"'.format( + self.config['id'] ) - migrations.migrate_paths(self.config) + ) + migrations.migrate_paths(self.config) if self.config['transport'].lower() == 'zeromq': # Late import so logging works correctly import salt.minion @@ -248,29 +245,18 @@ def start(self): NOTE: Run any required code before calling `super()`. ''' - reconnect = True - while reconnect: - reconnect = False - try: - self.prepare() - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - except SaltClientError as exc: - logger.error(exc) - if self.config.get('restart_on_error'): - logger.warn('** Restarting minion **') - s = randint(0, self.config.get('random_reauth_delay', 10)) - logger.info('Sleeping random_reauth_delay of {0} seconds'.format(s)) - time.sleep(s) - reconnect = True - finally: - self.shutdown() + try: + self.prepare() + if check_user(self.config['user']): + self.minion.tune_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + finally: + self.shutdown() def shutdown(self): ''' From 3758f9434b55967ba3fd132ddbc50b632cddb2ef Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 31 Jul 2014 20:50:46 -0700 Subject: [PATCH 286/769] Basics working, can't handle restart of a master yet... --- src/saltext/consul/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index b7b7287..4da5735 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -420,7 +420,11 @@ def prepare(self): # Late import so logging works correctly import salt.minion self.daemonize_if_required() - self.syndic = salt.minion.Syndic(self.config) + # if its a multisyndic, do so + if isinstance(self.config.get('master'), list): + self.syndic = salt.minion.MultiSyndic(self.config) + else: + self.syndic = salt.minion.Syndic(self.config) self.set_pidfile() def start(self): From 935ba255b7c4a78a6d8567c9e1117f66b172e9e9 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 22 Aug 2014 12:02:56 -0600 Subject: [PATCH 287/769] Stop using deprecated access to message attribute The message attribute has been deprecated. See: http://legacy.python.org/dev/peps/pep-0352/#retracted-ideas --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8e366d1..7978167 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -514,7 +514,7 @@ def ext_pillar(self, pillar, pillar_dirs): pillar_dirs, key) except TypeError as exc: - if exc.message.startswith('ext_pillar() takes exactly '): + if str(exc).startswith('ext_pillar() takes exactly '): log.warning('Deprecation warning: ext_pillar "{0}"' ' needs to accept minion_id as first' ' argument'.format(key)) From fc7ee3b97e00ffbbe76368b735b0a9268cee69c1 Mon Sep 17 00:00:00 2001 From: Scott Coil Date: Sun, 31 Aug 2014 16:46:22 -0500 Subject: [PATCH 288/769] add ability to target pillar from ext pillar --- src/saltext/consul/pillar/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7978167..f05a595 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -564,10 +564,19 @@ def compile_pillar(self, ext=True, pillar_dirs=None): Render the pillar data and return ''' top, terrors = self.get_top() - matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) if ext: - pillar = self.ext_pillar(pillar, pillar_dirs) + if self.opts.get('ext_pillar_first', False): + self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + matches = self.top_matches(top) + pillar, errors = self.render_pillar(matches) + pillar = self.merge_sources(pillar, self.opts['pillar']) + else: + matches = self.top_matches(top) + pillar, errors = self.render_pillar(matches) + pillar = self.ext_pillar(pillar, pillar_dirs) + else: + matches = self.top_matches(top) + pillar, errors = self.render_pillar(matches) errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) From 1c347f6fba92809040bf2a96e89c4f1e3b56043f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 13 Sep 2014 20:56:07 +0100 Subject: [PATCH 289/769] Filter the backports package UserWarning about being re-imported --- src/saltext/consul/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e994a28..d9bb846 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -25,6 +25,13 @@ DeprecationWarning ) +# Filter the backports package UserWarning about being re-imported +warnings.filterwarnings( + 'ignore', + '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + UserWarning +) + # Import salt libs # We import log ASAP because we NEED to make sure that any logger instance salt # instantiates is using salt.log.setup.SaltLoggingClass From 4e4abe357474ac4eee8ea81d19b56031ca69fa1f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Sat, 20 Sep 2014 12:20:16 -0600 Subject: [PATCH 290/769] Split the file clients, still use local for pillar --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7978167..53cc756 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -105,7 +105,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) - self.client = salt.fileclient.get_file_client(self.opts) + self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': opts['grains'] = grains From 8dc280cf95c5eace8a8862df1187ef9bb8b36a52 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 31 Jul 2014 20:50:46 -0700 Subject: [PATCH 291/769] Basics working, can't handle restart of a master yet... --- src/saltext/consul/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d9bb846..dd82a7d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -441,7 +441,11 @@ def prepare(self): # Late import so logging works correctly import salt.minion self.daemonize_if_required() - self.syndic = salt.minion.Syndic(self.config) + # if its a multisyndic, do so + if isinstance(self.config.get('master'), list): + self.syndic = salt.minion.MultiSyndic(self.config) + else: + self.syndic = salt.minion.Syndic(self.config) self.set_pidfile() def start(self): From 9a5f9955e39f6dece35d9057d684ace589649863 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 29 Sep 2014 13:41:46 -0600 Subject: [PATCH 292/769] Make renderers fully aware of pillar context --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 53cc756..4bebb95 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -378,7 +378,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state = None try: state = compile_template( - fn_, self.rend, self.opts['renderer'], saltenv, sls, **defaults) + fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults) except Exception as exc: msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( sls, exc From 672e4c07bbdf3593cb949983601404c22e9a5c0c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 1 Oct 2014 16:03:04 -0600 Subject: [PATCH 293/769] Don't send pillar render exception to minion Refs #16283 --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4bebb95..04f0bd4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -384,7 +384,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): sls, exc ) log.critical(msg) - errors.append(msg) + errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) mods.add(sls) nstate = None if state: From f90021612afab3633a8cf2f3115dfa685c4096e7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 1 Oct 2014 16:03:04 -0600 Subject: [PATCH 294/769] Don't send pillar render exception to minion Refs #16283 --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8635a31..d9ed2c6 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -384,7 +384,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): sls, exc ) log.critical(msg) - errors.append(msg) + errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) mods.add(sls) nstate = None if state: From 9b4323a2be2c8f8adf199a5164ad40fd9561ac1b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 10 Oct 2014 16:25:53 -0600 Subject: [PATCH 295/769] Pillar overwrite merge stategy --- src/saltext/consul/pillar/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 04f0bd4..ad3898d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -34,6 +34,13 @@ def merge_recurse(obj_a, obj_b): def merge_aggregate(obj_a, obj_b): return merge_recursive(obj_a, obj_b, level=1) +def merge_overwrite(obj_a, obj_b): + for obj in obj_b: + if obj in obj_a.keys(): + obj_a[obj] = obj_b[obj] + return obj_a + else: + return merge_recurse(obj_a, obj_b) def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): ''' @@ -552,6 +559,8 @@ def merge_sources(self, obj_a, obj_b): elif strategy == 'aggregate': #: level = 1 merge at least root data merged = merge_aggregate(obj_a, obj_b) + elif strategy == 'overwrite': + merged = merge_overwrite(obj_a, obj_b) else: log.warning('unknown merging strategy {0}, ' 'fallback to recurse'.format(strategy)) From 4c610674c34d8d0fafc0bacf76d7d340332df5bd Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Mon, 13 Oct 2014 09:25:14 -0600 Subject: [PATCH 296/769] lint --- src/saltext/consul/pillar/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ad3898d..b6de27e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -34,13 +34,14 @@ def merge_recurse(obj_a, obj_b): def merge_aggregate(obj_a, obj_b): return merge_recursive(obj_a, obj_b, level=1) + def merge_overwrite(obj_a, obj_b): for obj in obj_b: if obj in obj_a.keys(): obj_a[obj] = obj_b[obj] return obj_a - else: - return merge_recurse(obj_a, obj_b) + return merge_recurse(obj_a, obj_b) + def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): ''' From 9ab4a8e960ddeb0313a75dc946193e4d917c1091 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 31 Oct 2014 13:07:57 -0700 Subject: [PATCH 297/769] more cleanup no need to caste keys() to a list since it returns a list --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b6de27e..0e6fc0b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -37,7 +37,7 @@ def merge_aggregate(obj_a, obj_b): def merge_overwrite(obj_a, obj_b): for obj in obj_b: - if obj in obj_a.keys(): + if obj in obj_a: obj_a[obj] = obj_b[obj] return obj_a return merge_recurse(obj_a, obj_b) @@ -306,7 +306,7 @@ def merge_tops(self, tops): if isinstance(comp, string_types): states[comp] = True top[saltenv][tgt] = matches - top[saltenv][tgt].extend(list(states.keys())) + top[saltenv][tgt].extend(states.keys()) return self.sort_top_targets(top, orders) def sort_top_targets(self, top, orders): From ced82a11f0e888a82d7403c3b664117726715c78 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 31 Oct 2014 13:50:10 -0700 Subject: [PATCH 298/769] More dict cleanup keys() and values() create copies of the data in lists, iterators are better --- src/saltext/consul/pillar/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0e6fc0b..c3ba000 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -282,7 +282,7 @@ def merge_tops(self, tops): ''' top = collections.defaultdict(OrderedDict) orders = collections.defaultdict(OrderedDict) - for ctops in tops.values(): + for ctops in tops.itervalues(): for ctop in ctops: for saltenv, targets in ctop.items(): if saltenv == 'include': @@ -306,7 +306,7 @@ def merge_tops(self, tops): if isinstance(comp, string_types): states[comp] = True top[saltenv][tgt] = matches - top[saltenv][tgt].extend(states.keys()) + top[saltenv][tgt].extend(states) return self.sort_top_targets(top, orders) def sort_top_targets(self, top, orders): @@ -316,7 +316,7 @@ def sort_top_targets(self, top, orders): sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop for saltenv, targets in top.items(): - sorted_targets = sorted(targets.keys(), + sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: sorted_top[saltenv][target] = targets[target] From f57102be8f11dbc77cf6b3e2c6185cb1b03f22be Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 6 Nov 2014 20:19:33 -0700 Subject: [PATCH 299/769] More scaffolding support for Raet Salt Caller with Multi master --- src/saltext/consul/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 187ecfb..cfb7b00 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -265,6 +265,29 @@ def start(self): finally: self.shutdown() + def call(self): + ''' + Start the actual minion as a caller minion. + + If sub-classed, don't **ever** forget to run: + + super(YourSubClass, self).start() + + NOTE: Run any required code before calling `super()`. + ''' + try: + self.prepare() + if check_user(self.config['user']): + self.minion.call_in() + except (KeyboardInterrupt, SaltSystemExit) as exc: + logger.warn('Stopping the Salt Minion') + if isinstance(exc, KeyboardInterrupt): + logger.warn('Exiting on Ctrl-c') + else: + logger.error(str(exc)) + finally: + self.shutdown() + def shutdown(self): ''' If sub-classed, run any shutdown operations on this method. From ba2a0554fed9ed7db46f9f8c0e5f2915c8d63677 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 7 Nov 2014 09:51:25 -0700 Subject: [PATCH 300/769] make salt package importable in python3 --- src/saltext/consul/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 187ecfb..66a42a5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -3,6 +3,8 @@ Make me some salt! ''' +from __future__ import absolute_import + # Import python libs import os import sys @@ -206,7 +208,7 @@ def prepare(self): 'udp://', 'file://')): # Logfile is not using Syslog, verify - current_umask = os.umask(0077) + current_umask = os.umask(0o077) verify_files([logfile], self.config['user']) os.umask(current_umask) except OSError as err: From 1e9d1c7827f24b201dd2b099f6087b1cb959e85c Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 7 Nov 2014 09:51:25 -0700 Subject: [PATCH 301/769] make salt package importable in python3 --- src/saltext/consul/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 187ecfb..66a42a5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -3,6 +3,8 @@ Make me some salt! ''' +from __future__ import absolute_import + # Import python libs import os import sys @@ -206,7 +208,7 @@ def prepare(self): 'udp://', 'file://')): # Logfile is not using Syslog, verify - current_umask = os.umask(0077) + current_umask = os.umask(0o077) verify_files([logfile], self.config['user']) os.umask(current_umask) except OSError as err: From 9426a5a36823432bcdb06a9638e1f69b6ec084fb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 8 Nov 2014 09:22:52 -0700 Subject: [PATCH 302/769] Added support for caller kind opts['__role'] for raet salt caller --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cfb7b00..9024a5f 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -40,6 +40,7 @@ # cause the build to fail from salt.version import __version__ from salt.utils import migrations +from salt.utils import kinds try: from salt.utils import parsers, ip_bracket @@ -278,6 +279,7 @@ def call(self): try: self.prepare() if check_user(self.config['user']): + self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] self.minion.call_in() except (KeyboardInterrupt, SaltSystemExit) as exc: logger.warn('Stopping the Salt Minion') From 1bbbf3ee57947bbaa870370f80bcd16870c096cd Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sat, 8 Nov 2014 18:19:46 -0700 Subject: [PATCH 303/769] If we already have funcs generated, let the pillar use them This saves an average of ~0.09s per salt-call --- src/saltext/consul/pillar/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a380fde..7ec1767 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -43,7 +43,7 @@ def merge_overwrite(obj_a, obj_b): return merge_recurse(obj_a, obj_b) -def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): +def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -59,14 +59,14 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None): return { 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], Pillar)(opts, grains, id_, saltenv, ext) + }.get(opts['file_client'], Pillar)(opts, grains, id_, saltenv, ext, functions=funcs) class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None): + def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext From b92dfb3bd730b85abab0063879c19b24e8496d05 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Tue, 11 Nov 2014 12:32:58 -0800 Subject: [PATCH 304/769] replace salt._compat.string_types with six.string_types --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7ec1767..5efaab2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -15,7 +15,7 @@ import salt.minion import salt.crypt import salt.transport -from salt._compat import string_types +from six import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.serializers.yamlex import merge_recursive From eca72fc6713de2b165793b9e6533c6556817ccf1 Mon Sep 17 00:00:00 2001 From: Nitin Madhok Date: Tue, 11 Nov 2014 16:51:21 -0500 Subject: [PATCH 305/769] Making salt.pillar.__init__ python3 compatible --- src/saltext/consul/pillar/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7ec1767..39b06ca 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -2,6 +2,7 @@ ''' Render the pillar data ''' +from __future__ import absolute_import # Import python libs import os @@ -21,6 +22,7 @@ from salt.utils.serializers.yamlex import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ +import six log = logging.getLogger(__name__) @@ -236,7 +238,7 @@ def get_tops(self): .format(exc))) # Search initial top files for includes - for saltenv, ctops in tops.items(): + for saltenv, ctops in list(list(tops.items())): for ctop in ctops: if 'include' not in ctop: continue @@ -246,7 +248,7 @@ def get_tops(self): # Go through the includes and pull out the extra tops and add them while include: pops = [] - for saltenv, states in include.items(): + for saltenv, states in list(list(include.items())): pops.append(saltenv) if not states: continue @@ -282,9 +284,9 @@ def merge_tops(self, tops): ''' top = collections.defaultdict(OrderedDict) orders = collections.defaultdict(OrderedDict) - for ctops in tops.itervalues(): + for ctops in six.itervalues(tops): for ctop in ctops: - for saltenv, targets in ctop.items(): + for saltenv, targets in list(list(ctop.items())): if saltenv == 'include': continue for tgt in targets: @@ -315,7 +317,7 @@ def sort_top_targets(self, top, orders): ''' sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop - for saltenv, targets in top.items(): + for saltenv, targets in list(list(top.items())): sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: @@ -344,11 +346,11 @@ def top_matches(self, top): {'saltenv': ['state1', 'state2', ...]} ''' matches = {} - for saltenv, body in top.items(): + for saltenv, body in list(list(top.items())): if self.opts['environment']: if saltenv != self.opts['environment']: continue - for match, data in body.items(): + for match, data in list(list(body.items())): if self.matcher.confirm_top( match, data, @@ -410,7 +412,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): - sub_sls, v = sub_sls.iteritems().next() + sub_sls, v = next(sub_sls.iteritems()) defaults = v.get('defaults', {}) key = v.get('key', None) else: @@ -441,7 +443,7 @@ def render_pillar(self, matches): ''' pillar = {} errors = [] - for saltenv, pstates in matches.items(): + for saltenv, pstates in list(list(matches.items())): mods = set() for sls in pstates: pstate, mods, err = self.render_pstate(sls, saltenv, mods) @@ -509,7 +511,7 @@ def ext_pillar(self, pillar, pillar_dirs): if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} - for key, val in run.items(): + for key, val in list(list(run.items())): if key not in self.ext_pillars: err = ('Specified ext_pillar interface {0} is ' 'unavailable').format(key) From 853d5270e09ad4df01189a2a42889c6eb861ef50 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Tue, 11 Nov 2014 12:32:58 -0800 Subject: [PATCH 306/769] replace salt._compat.string_types with six.string_types --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7ec1767..5efaab2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -15,7 +15,7 @@ import salt.minion import salt.crypt import salt.transport -from salt._compat import string_types +from six import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.serializers.yamlex import merge_recursive From e99bb1c1a3f82e75d8b64440ea3c1e43be3d674d Mon Sep 17 00:00:00 2001 From: Nitin Madhok Date: Tue, 11 Nov 2014 18:40:55 -0500 Subject: [PATCH 307/769] Removing lists and change back to earlier --- src/saltext/consul/pillar/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 39b06ca..3d17802 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -238,7 +238,7 @@ def get_tops(self): .format(exc))) # Search initial top files for includes - for saltenv, ctops in list(list(tops.items())): + for saltenv, ctops in tops.items(): for ctop in ctops: if 'include' not in ctop: continue @@ -248,7 +248,7 @@ def get_tops(self): # Go through the includes and pull out the extra tops and add them while include: pops = [] - for saltenv, states in list(list(include.items())): + for saltenv, states in include.items(): pops.append(saltenv) if not states: continue @@ -286,7 +286,7 @@ def merge_tops(self, tops): orders = collections.defaultdict(OrderedDict) for ctops in six.itervalues(tops): for ctop in ctops: - for saltenv, targets in list(list(ctop.items())): + for saltenv, targets in ctop.items(): if saltenv == 'include': continue for tgt in targets: @@ -317,7 +317,7 @@ def sort_top_targets(self, top, orders): ''' sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop - for saltenv, targets in list(list(top.items())): + for saltenv, targets in top.items(): sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: @@ -346,11 +346,11 @@ def top_matches(self, top): {'saltenv': ['state1', 'state2', ...]} ''' matches = {} - for saltenv, body in list(list(top.items())): + for saltenv, body in top.items(): if self.opts['environment']: if saltenv != self.opts['environment']: continue - for match, data in list(list(body.items())): + for match, data in body.items(): if self.matcher.confirm_top( match, data, @@ -443,7 +443,7 @@ def render_pillar(self, matches): ''' pillar = {} errors = [] - for saltenv, pstates in list(list(matches.items())): + for saltenv, pstates in matches.items(): mods = set() for sls in pstates: pstate, mods, err = self.render_pstate(sls, saltenv, mods) @@ -511,7 +511,7 @@ def ext_pillar(self, pillar, pillar_dirs): if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} - for key, val in list(list(run.items())): + for key, val in run.items(): if key not in self.ext_pillars: err = ('Specified ext_pillar interface {0} is ' 'unavailable').format(key) From 97574d672b1fb8f83a87e4b78732fc55737a434e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 12 Nov 2014 15:11:16 -0700 Subject: [PATCH 308/769] Fix race condition with Raet Salt Caller where the minion caller could cleanup (erase) the cli salt-call lanestack --- src/saltext/consul/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a1c5080..dfda894 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -268,10 +268,13 @@ def start(self): finally: self.shutdown() - def call(self): + def call(self, cleanup_protecteds): ''' Start the actual minion as a caller minion. + cleanup_protecteds is list of yard host addresses that should not be + cleaned up this is to fix race condition when salt-caller minion starts up + If sub-classed, don't **ever** forget to run: super(YourSubClass, self).start() @@ -282,6 +285,7 @@ def call(self): self.prepare() if check_user(self.config['user']): self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] + self.minion.opts['raet_cleanup_protecteds'] = cleanup_protecteds self.minion.call_in() except (KeyboardInterrupt, SaltSystemExit) as exc: logger.warn('Stopping the Salt Minion') From e3e0fad755ebe8b553b040874aeeb4abf5493361 Mon Sep 17 00:00:00 2001 From: benosman Date: Thu, 6 Nov 2014 15:51:39 +0000 Subject: [PATCH 309/769] Read Pillar files into OrderedDict to preserve source order --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4bc5e8c..e5f353a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -441,7 +441,7 @@ def render_pillar(self, matches): Extract the sls pillar files from the matches and render them into the pillar ''' - pillar = {} + pillar = OrderedDict() errors = [] for saltenv, pstates in matches.items(): mods = set() @@ -578,7 +578,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): top, terrors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + self.opts['pillar'] = self.ext_pillar(OrderedDict(), pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) pillar = self.merge_sources(pillar, self.opts['pillar']) From f3c3d8c5a40632de0d0e93adcb34de5d4825dc76 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 18 Nov 2014 09:11:27 -0700 Subject: [PATCH 310/769] Revert "Read Pillar files into OrderedDict to preserve source order" --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e5f353a..4bc5e8c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -441,7 +441,7 @@ def render_pillar(self, matches): Extract the sls pillar files from the matches and render them into the pillar ''' - pillar = OrderedDict() + pillar = {} errors = [] for saltenv, pstates in matches.items(): mods = set() @@ -578,7 +578,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): top, terrors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'] = self.ext_pillar(OrderedDict(), pillar_dirs) + self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) pillar = self.merge_sources(pillar, self.opts['pillar']) From c2bb0eef9621f8e3098131c1b87886cf835b7a6c Mon Sep 17 00:00:00 2001 From: Nitin Madhok Date: Tue, 18 Nov 2014 00:28:52 -0500 Subject: [PATCH 311/769] Replaced import six in file /salt/pillar/__init__.py --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4bc5e8c..2741a42 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -22,7 +22,7 @@ from salt.utils.serializers.yamlex import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ -import six +import salt.utils.six as six log = logging.getLogger(__name__) From 03479a93b122825f41eb4082b7c564745cb66e0e Mon Sep 17 00:00:00 2001 From: Nitin Madhok Date: Tue, 18 Nov 2014 00:44:31 -0500 Subject: [PATCH 312/769] Replaced module six in file /salt/pillar/__init__.py --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 2741a42..8ff9aeb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -16,7 +16,7 @@ import salt.minion import salt.crypt import salt.transport -from six import string_types +from salt.utils.six import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.serializers.yamlex import merge_recursive From dc4697f90c44dfd14487a2d4a44fe78f329a5926 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 18 Nov 2014 18:40:02 +0000 Subject: [PATCH 313/769] Move `salt.utils.six` to `salt.ext.six` --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8ff9aeb..92f611b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -16,13 +16,13 @@ import salt.minion import salt.crypt import salt.transport -from salt.utils.six import string_types +from salt.ext.six import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.serializers.yamlex import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ -import salt.utils.six as six +import salt.ext.six as six log = logging.getLogger(__name__) From 4a7d5cbefc699a1bc7f4454745c7f72e4688cc8e Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 4 Dec 2014 08:34:35 -0800 Subject: [PATCH 314/769] More Sreq cleanup Changing the local variable names in the last few places to be consistent --- src/saltext/consul/pillar/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 92f611b..6df4e25 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -75,7 +75,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.grains = grains self.id_ = id_ self.serial = salt.payload.Serial(self.opts) - self.sreq = salt.transport.Channel.factory(opts) + self.channel = salt.transport.Channel.factory(opts) # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): @@ -89,8 +89,7 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - # ret = self.sreq.send(load, tries=3, timeout=7200) - ret_pillar = self.sreq.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) + ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) # key = self.auth.get_keys() # aes = key.private_decrypt(ret['key'], 4) From 82f94480d0264e1120f53cb5e36ff765b2001ec2 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 11 Dec 2014 14:40:58 -0700 Subject: [PATCH 315/769] Refactor salt/__init__ into cli/daemons and late-load outputters If we late-load the outputter, we can get to the pub slightly faster --- src/saltext/consul/__init__.py | 489 --------------------------------- 1 file changed, 489 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index dfda894..e69de29 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,489 +0,0 @@ -# coding: utf-8 -*- -''' -Make me some salt! -''' - -from __future__ import absolute_import - -# Import python libs -import os -import sys -import warnings - -# All salt related deprecation warnings should be shown once each! -warnings.filterwarnings( - 'once', # Show once - '', # No deprecation message match - DeprecationWarning, # This filter is for DeprecationWarnings - r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' -) - -# While we are supporting Python2.6, hide nested with-statements warnings -warnings.filterwarnings( - 'ignore', - 'With-statements now directly support multiple context managers', - DeprecationWarning -) - -# Filter the backports package UserWarning about being re-imported -warnings.filterwarnings( - 'ignore', - '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', - UserWarning -) - -# Import salt libs -# We import log ASAP because we NEED to make sure that any logger instance salt -# instantiates is using salt.log.setup.SaltLoggingClass -import salt.log.setup - - -# the try block below bypasses an issue at build time so that modules don't -# cause the build to fail -from salt.version import __version__ -from salt.utils import migrations -from salt.utils import kinds - -try: - from salt.utils import parsers, ip_bracket - from salt.utils.verify import check_user, verify_env, verify_socket - from salt.utils.verify import verify_files -except ImportError as exc: - if exc.args[0] != 'No module named _msgpack': - raise -from salt.exceptions import SaltSystemExit, MasterExit - - -# Let's instantiate logger using salt.log.setup.logging.getLogger() so pylint -# leaves us alone and stops complaining about an un-used import -logger = salt.log.setup.logging.getLogger(__name__) - - -class Master(parsers.MasterOptionParser): - ''' - Creates a master server - ''' - def prepare(self): - ''' - Run the preparation sequence required to start a salt master server. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).prepare() - ''' - self.parse_args() - - try: - if self.config['verify_env']: - v_dirs = [ - self.config['pki_dir'], - os.path.join(self.config['pki_dir'], 'minions'), - os.path.join(self.config['pki_dir'], 'minions_pre'), - os.path.join(self.config['pki_dir'], 'minions_denied'), - os.path.join(self.config['pki_dir'], - 'minions_autosign'), - os.path.join(self.config['pki_dir'], - 'minions_rejected'), - self.config['cachedir'], - os.path.join(self.config['cachedir'], 'jobs'), - os.path.join(self.config['cachedir'], 'proc'), - self.config['sock_dir'], - self.config['token_dir'], - self.config['sqlite_queue_dir'], - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], - ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - verify_files([logfile], self.config['user']) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info('Setting up the Salt Master') - - if self.config['transport'].lower() == 'zeromq': - if not verify_socket(self.config['interface'], - self.config['publish_port'], - self.config['ret_port']): - self.exit(4, 'The ports are not available to bind\n') - self.config['interface'] = ip_bracket(self.config['interface']) - migrations.migrate_paths(self.config) - - # Late import so logging works correctly - import salt.master - self.master = salt.master.Master(self.config) - else: - # Add a udp port check here - import salt.daemons.flo - self.master = salt.daemons.flo.IofloMaster(self.config) - self.daemonize_if_required() - self.set_pidfile() - - def start(self): - ''' - Start the actual master. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).start() - - NOTE: Run any required code before calling `super()`. - ''' - self.prepare() - if check_user(self.config['user']): - self.master.start() - - def shutdown(self): - ''' - If sub-classed, run any shutdown operations on this method. - ''' - - -class Minion(parsers.MinionOptionParser): - ''' - Create a minion server - ''' - def prepare(self): - ''' - Run the preparation sequence required to start a salt minion. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).prepare() - ''' - self.parse_args() - - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) - confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd - ) - else: - confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' - ) - v_dirs = [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ] - if self.config.get('transport') == 'raet': - v_dirs.append(os.path.join(self.config['pki_dir'], 'accepted')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'pending')) - v_dirs.append(os.path.join(self.config['pki_dir'], 'rejected')) - v_dirs.append(os.path.join(self.config['cachedir'], 'raet')) - verify_env( - v_dirs, - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], - ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - current_umask = os.umask(0o077) - verify_files([logfile], self.config['user']) - os.umask(current_umask) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Minion "{0}"'.format( - self.config['id'] - ) - ) - migrations.migrate_paths(self.config) - if self.config['transport'].lower() == 'zeromq': - # Late import so logging works correctly - import salt.minion - # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion could halt - # the boot process waiting for a key to be accepted on the master. - # This is the latest safe place to daemonize - self.daemonize_if_required() - self.set_pidfile() - if isinstance(self.config.get('master'), list): - if self.config.get('master_type') == 'failover': - self.minion = salt.minion.Minion(self.config) - else: - self.minion = salt.minion.MultiMinion(self.config) - else: - self.minion = salt.minion.Minion(self.config) - else: - import salt.daemons.flo - self.daemonize_if_required() - self.set_pidfile() - self.minion = salt.daemons.flo.IofloMinion(self.config) - - def start(self): - ''' - Start the actual minion. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).start() - - NOTE: Run any required code before calling `super()`. - ''' - try: - self.prepare() - if check_user(self.config['user']): - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - finally: - self.shutdown() - - def call(self, cleanup_protecteds): - ''' - Start the actual minion as a caller minion. - - cleanup_protecteds is list of yard host addresses that should not be - cleaned up this is to fix race condition when salt-caller minion starts up - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).start() - - NOTE: Run any required code before calling `super()`. - ''' - try: - self.prepare() - if check_user(self.config['user']): - self.minion.opts['__role'] = kinds.APPL_KIND_NAMES[kinds.applKinds.caller] - self.minion.opts['raet_cleanup_protecteds'] = cleanup_protecteds - self.minion.call_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - finally: - self.shutdown() - - def shutdown(self): - ''' - If sub-classed, run any shutdown operations on this method. - ''' - - -class ProxyMinion(parsers.MinionOptionParser): - ''' - Create a proxy minion server - ''' - def prepare(self, proxydetails): - ''' - Run the preparation sequence required to start a salt minion. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).prepare() - ''' - self.parse_args() - - try: - if self.config['verify_env']: - confd = self.config.get('default_include') - if confd: - # If 'default_include' is specified in config, then use it - if '*' in confd: - # Value is of the form "minion.d/*.conf" - confd = os.path.dirname(confd) - if not os.path.isabs(confd): - # If configured 'default_include' is not an absolute - # path, consider it relative to folder of 'conf_file' - # (/etc/salt by default) - confd = os.path.join( - os.path.dirname(self.config['conf_file']), confd - ) - else: - confd = os.path.join( - os.path.dirname(self.config['conf_file']), 'minion.d' - ) - verify_env( - [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - confd, - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], - ) - if 'proxy_log' in proxydetails: - logfile = proxydetails['proxy_log'] - else: - logfile = None - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - verify_files([logfile], self.config['user']) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.config['proxy'] = proxydetails - self.setup_logfile_logger() - logger.info( - 'Setting up a Salt Proxy Minion "{0}"'.format( - self.config['id'] - ) - ) - migrations.migrate_paths(self.config) - # Late import so logging works correctly - import salt.minion - # If the minion key has not been accepted, then Salt enters a loop - # waiting for it, if we daemonize later then the minion could halt - # the boot process waiting for a key to be accepted on the master. - # This is the latest safe place to daemonize - self.daemonize_if_required() - self.set_pidfile() - if isinstance(self.config.get('master'), list): - self.minion = salt.minion.MultiMinion(self.config) - else: - self.minion = salt.minion.ProxyMinion(self.config) - - def start(self, proxydetails): - ''' - Start the actual minion. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).start() - - NOTE: Run any required code before calling `super()`. - ''' - self.prepare(proxydetails) - try: - self.minion.tune_in() - except (KeyboardInterrupt, SaltSystemExit) as exc: - logger.warn('Stopping the Salt Proxy Minion') - if isinstance(exc, KeyboardInterrupt): - logger.warn('Exiting on Ctrl-c') - else: - logger.error(str(exc)) - finally: - self.shutdown() - - def shutdown(self): - ''' - If sub-classed, run any shutdown operations on this method. - ''' - if 'proxy' in self.minion.opts: - self.minion.opts['proxyobject'].shutdown(self.minion.opts) - - -class Syndic(parsers.SyndicOptionParser): - ''' - Create a syndic server - ''' - - def prepare(self): - ''' - Run the preparation sequence required to start a salt syndic minion. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).prepare() - ''' - self.parse_args() - try: - if self.config['verify_env']: - verify_env( - [ - self.config['pki_dir'], - self.config['cachedir'], - self.config['sock_dir'], - self.config['extension_modules'], - ], - self.config['user'], - permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], - ) - logfile = self.config['log_file'] - if logfile is not None and not logfile.startswith(('tcp://', - 'udp://', - 'file://')): - # Logfile is not using Syslog, verify - verify_files([logfile], self.config['user']) - except OSError as err: - logger.exception('Failed to prepare salt environment') - sys.exit(err.errno) - - self.setup_logfile_logger() - logger.info( - 'Setting up the Salt Syndic Minion "{0}"'.format( - self.config['id'] - ) - ) - - # Late import so logging works correctly - import salt.minion - self.daemonize_if_required() - # if its a multisyndic, do so - if isinstance(self.config.get('master'), list): - self.syndic = salt.minion.MultiSyndic(self.config) - else: - self.syndic = salt.minion.Syndic(self.config) - self.set_pidfile() - - def start(self): - ''' - Start the actual syndic. - - If sub-classed, don't **ever** forget to run: - - super(YourSubClass, self).start() - - NOTE: Run any required code before calling `super()`. - ''' - self.prepare() - if check_user(self.config['user']): - try: - self.syndic.tune_in() - except KeyboardInterrupt: - logger.warn('Stopping the Salt Syndic Minion') - self.shutdown() - - def shutdown(self): - ''' - If sub-classed, run any shutdown operations on this method. - ''' From 1f28e7e89790396dd7a2c189ad01333608410bfb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 11 Dec 2014 15:57:33 -0700 Subject: [PATCH 316/769] Lint --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e69de29..3f2371e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -0,0 +1,4 @@ + # -*- coding: utf-8 -*- + ''' + Salt package + ''' From 2b7f9960b798f59a2689fde350ff2dda22f557a5 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 11 Dec 2014 16:30:43 -0700 Subject: [PATCH 317/769] Oopslint --- src/saltext/consul/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3f2371e..1b41e76 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,4 +1,4 @@ - # -*- coding: utf-8 -*- - ''' - Salt package - ''' +# -*- coding: utf-8 -*- +''' +Salt package +''' From 12979f6e1764c286dc0bd4ac37ff87d10c4fb680 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 15 Dec 2014 10:59:08 -0700 Subject: [PATCH 318/769] Restore warnings --- src/saltext/consul/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 1b41e76..d7efe27 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,3 +2,25 @@ ''' Salt package ''' +import warnings +# All salt related deprecation warnings should be shown once each! +warnings.filterwarnings( + 'once', # Show once + '', # No deprecation message match + DeprecationWarning, # This filter is for DeprecationWarnings + r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' +) + +# While we are supporting Python2.6, hide nested with-statements warnings +warnings.filterwarnings( + 'ignore', + 'With-statements now directly support multiple context managers', + DeprecationWarning +) + +# Filter the backports package UserWarning about being re-imported +warnings.filterwarnings( + 'ignore', + '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + UserWarning +) From 6995dcad20fb20a06196e72be22a84fbf2cff8fd Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 1 Jan 2015 11:13:13 -0800 Subject: [PATCH 319/769] Cleanup of old comments, and style --- src/saltext/consul/pillar/__init__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6df4e25..17056a1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -74,9 +74,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.ext = ext self.grains = grains self.id_ = id_ - self.serial = salt.payload.Serial(self.opts) self.channel = salt.transport.Channel.factory(opts) - # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): ''' @@ -89,12 +87,11 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) - - # key = self.auth.get_keys() - # aes = key.private_decrypt(ret['key'], 4) - # pcrypt = salt.crypt.Crypticle(self.opts, aes) - # ret_pillar = pcrypt.loads(ret['pillar']) + ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, + dictkey='pillar', + tries=3, + timeout=7200, + ) if not isinstance(ret_pillar, dict): log.error( From a85cd81b36e924e889417b97c34de776651e7e92 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 5 Jan 2015 17:21:54 -0800 Subject: [PATCH 320/769] Pop "aes" from the opts before passing to ext_pillar modules We should probably not put "aes" in the global opts dict, but rather as a class attribute of auth/crypticle or something --- src/saltext/consul/pillar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6df4e25..98a3895 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -134,6 +134,9 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots + # TODO: consolidate into "sanitize opts" + if 'aes' in ext_pillar_opts: + ext_pillar_opts.pop('aes') self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -591,6 +594,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) + # TODO: consolidate into sanitize function if 'grains' in mopts: mopts.pop('grains') if 'aes' in mopts: From 0a0c70f4b1fc2983ca1c953c4ae21ca209dd9060 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 6 Jan 2015 19:58:05 -0800 Subject: [PATCH 321/769] Add new ignore_missing option to pillar top --- src/saltext/consul/pillar/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 98a3895..9262e43 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -142,6 +142,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.merge_strategy = opts['pillar_source_merging_strategy'] self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) + self.ignored_pillars = {} def __valid_ext(self, ext): ''' @@ -295,6 +296,7 @@ def merge_tops(self, tops): matches = [] states = OrderedDict() orders[saltenv][tgt] = 0 + ignore_missing = False for comp in ctop[saltenv][tgt]: if isinstance(comp, dict): if 'match' in comp: @@ -307,8 +309,12 @@ def merge_tops(self, tops): except ValueError: order = 0 orders[saltenv][tgt] = order + if comp.get('ignore_missing', False): + ignore_missing = True if isinstance(comp, string_types): states[comp] = True + if ignore_missing: + self.ignored_pillars[saltenv] = states.keys() top[saltenv][tgt] = matches top[saltenv][tgt].extend(states) return self.sort_top_targets(top, orders) @@ -375,7 +381,11 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): errors = [] fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: - if self.opts['pillar_roots'].get(saltenv): + if sls in self.ignored_pillars.get(saltenv, []): + log.debug('Skipping ignored and missing SLS {0!r} in' + ' environment {1!r}'.format(sls, saltenv)) + return None, mods, errors + elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS {0!r} in environment {1!r} is not' ' available on the salt master').format(sls, saltenv) log.error(msg) From f51c0e43eeb6f4dfd44686abc68b149c7c341641 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 8 Jan 2015 14:57:04 -0700 Subject: [PATCH 322/769] Revert "Cleanup and performance tweaks" --- src/saltext/consul/pillar/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c2cf9ab..98a3895 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -74,7 +74,9 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.ext = ext self.grains = grains self.id_ = id_ + self.serial = salt.payload.Serial(self.opts) self.channel = salt.transport.Channel.factory(opts) + # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): ''' @@ -87,11 +89,12 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, - dictkey='pillar', - tries=3, - timeout=7200, - ) + ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) + + # key = self.auth.get_keys() + # aes = key.private_decrypt(ret['key'], 4) + # pcrypt = salt.crypt.Crypticle(self.opts, aes) + # ret_pillar = pcrypt.loads(ret['pillar']) if not isinstance(ret_pillar, dict): log.error( From 51e65cbc138b5b00f47997e19360b2323d18e1dc Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 8 Jan 2015 14:38:17 -0800 Subject: [PATCH 323/769] Revert "Revert "Cleanup and performance tweaks"" This reverts commit f51c0e43eeb6f4dfd44686abc68b149c7c341641. --- src/saltext/consul/pillar/__init__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 98a3895..c2cf9ab 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -74,9 +74,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.ext = ext self.grains = grains self.id_ = id_ - self.serial = salt.payload.Serial(self.opts) self.channel = salt.transport.Channel.factory(opts) - # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): ''' @@ -89,12 +87,11 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) - - # key = self.auth.get_keys() - # aes = key.private_decrypt(ret['key'], 4) - # pcrypt = salt.crypt.Crypticle(self.opts, aes) - # ret_pillar = pcrypt.loads(ret['pillar']) + ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, + dictkey='pillar', + tries=3, + timeout=7200, + ) if not isinstance(ret_pillar, dict): log.error( From 6617c83815a10197bee35b8fa74c0d3ea3cf9692 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 8 Jan 2015 17:44:16 -0600 Subject: [PATCH 324/769] Add a __pillar key to opts --- src/saltext/consul/pillar/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c3ba000..faf407e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -165,6 +165,7 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): saltenv = env opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] + opts['__pillar'] = True opts['file_client'] = 'local' if not grains: opts['grains'] = {} From 2da4c804c1fa18dc391089af71099fb8dbc43059 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Mon, 5 Jan 2015 17:21:54 -0800 Subject: [PATCH 325/769] Pop "aes" from the opts before passing to ext_pillar modules We should probably not put "aes" in the global opts dict, but rather as a class attribute of auth/crypticle or something --- src/saltext/consul/pillar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8a155a7..3d01785 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -134,6 +134,9 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots + # TODO: consolidate into "sanitize opts" + if 'aes' in ext_pillar_opts: + ext_pillar_opts.pop('aes') self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -592,6 +595,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) + # TODO: consolidate into sanitize function if 'grains' in mopts: mopts.pop('grains') if 'aes' in mopts: From 3c345024397b1889ef74bcd8fea7a85de485b4a2 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 8 Jan 2015 14:38:17 -0800 Subject: [PATCH 326/769] Revert "Revert "Cleanup and performance tweaks"" This reverts commit f51c0e43eeb6f4dfd44686abc68b149c7c341641. --- src/saltext/consul/pillar/__init__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3d01785..3081fcf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -74,9 +74,7 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.ext = ext self.grains = grains self.id_ = id_ - self.serial = salt.payload.Serial(self.opts) self.channel = salt.transport.Channel.factory(opts) - # self.auth = salt.crypt.SAuth(opts) def compile_pillar(self): ''' @@ -89,12 +87,11 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', tries=3, timeout=7200) - - # key = self.auth.get_keys() - # aes = key.private_decrypt(ret['key'], 4) - # pcrypt = salt.crypt.Crypticle(self.opts, aes) - # ret_pillar = pcrypt.loads(ret['pillar']) + ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, + dictkey='pillar', + tries=3, + timeout=7200, + ) if not isinstance(ret_pillar, dict): log.error( From ef30e13429aa81e620fbd0bed4b1d7690ebaf520 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Wed, 14 Jan 2015 21:22:40 -0800 Subject: [PATCH 327/769] Remove *insane* 7200s timeout (2h). If the pillar isn't back in the regular 1m its probably busted, if not we can set this timeout to something else, but 2h seems excessive. I've found a few minions stuck in this code block, since the minion's main process can get stuck here waiting for 2 hours. --- src/saltext/consul/pillar/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3081fcf..4e7e4b7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -89,8 +89,6 @@ def compile_pillar(self): load['ext'] = self.ext ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, dictkey='pillar', - tries=3, - timeout=7200, ) if not isinstance(ret_pillar, dict): From acfb9755481c8bf96451512d2456677ed60723c3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 20 Jan 2015 13:56:34 -0700 Subject: [PATCH 328/769] Add an error to the master log for failed pillar builds --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b035cbd..c0d33e4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -235,6 +235,8 @@ def get_tops(self): errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) + log.error('Pillar rendering failed for minion {0}: '.format(self.opts['id']), + exc_info=True) # Search initial top files for includes for saltenv, ctops in tops.items(): From 6566e03f293476edcdea393e7d9b4af83ea6c035 Mon Sep 17 00:00:00 2001 From: Clint Armstrong Date: Wed, 21 Jan 2015 13:07:19 -0500 Subject: [PATCH 329/769] Refactored git repo to host both states and pillars From cd2ed1d24068cf00a0f81eaf7a9102975f223844 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 23 Jan 2015 09:48:58 -0700 Subject: [PATCH 330/769] Remove __pillar completely --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index faf407e..c3ba000 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -165,7 +165,6 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): saltenv = env opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] - opts['__pillar'] = True opts['file_client'] = 'local' if not grains: opts['grains'] = {} From ac184c28914a48af41efbc1e9cf990f9f4a602a8 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 29 Jan 2015 18:10:56 -0800 Subject: [PATCH 331/769] Remove various AES sanitization attempts. --- src/saltext/consul/pillar/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4e7e4b7..5deb926 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -129,9 +129,6 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots - # TODO: consolidate into "sanitize opts" - if 'aes' in ext_pillar_opts: - ext_pillar_opts.pop('aes') self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -590,11 +587,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): errors.extend(terrors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) - # TODO: consolidate into sanitize function if 'grains' in mopts: mopts.pop('grains') - if 'aes' in mopts: - mopts.pop('aes') # Restore the actual file_roots path. Issue 5449 mopts['file_roots'] = self.actual_file_roots mopts['saltversion'] = __version__ From bad044c603ca9d464324a70132c51027193ec9c7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 22 Nov 2014 14:31:01 +0000 Subject: [PATCH 332/769] Py3 compatibility fixes. --- src/saltext/consul/pillar/__init__.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0de25ab..e22fb2a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -2,9 +2,9 @@ ''' Render the pillar data ''' -from __future__ import absolute_import # Import python libs +from __future__ import absolute_import import os import collections import logging @@ -16,12 +16,13 @@ import salt.minion import salt.crypt import salt.transport -from salt.ext.six import string_types from salt.template import compile_template from salt.utils.dictupdate import update from salt.utils.serializers.yamlex import merge_recursive from salt.utils.odict import OrderedDict from salt.version import __version__ + +# Import 3rd-party libs import salt.ext.six as six @@ -238,7 +239,7 @@ def get_tops(self): exc_info=True) # Search initial top files for includes - for saltenv, ctops in tops.items(): + for saltenv, ctops in six.iteritems(tops): for ctop in ctops: if 'include' not in ctop: continue @@ -248,7 +249,7 @@ def get_tops(self): # Go through the includes and pull out the extra tops and add them while include: pops = [] - for saltenv, states in include.items(): + for saltenv, states in six.iteritems(include): pops.append(saltenv) if not states: continue @@ -286,7 +287,7 @@ def merge_tops(self, tops): orders = collections.defaultdict(OrderedDict) for ctops in six.itervalues(tops): for ctop in ctops: - for saltenv, targets in ctop.items(): + for saltenv, targets in six.iteritems(ctop): if saltenv == 'include': continue for tgt in targets: @@ -308,7 +309,7 @@ def merge_tops(self, tops): orders[saltenv][tgt] = order if comp.get('ignore_missing', False): ignore_missing = True - if isinstance(comp, string_types): + if isinstance(comp, six.string_types): states[comp] = True if ignore_missing: self.ignored_pillars[saltenv] = states.keys() @@ -322,7 +323,7 @@ def sort_top_targets(self, top, orders): ''' sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop - for saltenv, targets in top.items(): + for saltenv, targets in six.iteritems(top): sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: @@ -351,11 +352,11 @@ def top_matches(self, top): {'saltenv': ['state1', 'state2', ...]} ''' matches = {} - for saltenv, body in top.items(): + for saltenv, body in six.iteritems(top): if self.opts['environment']: if saltenv != self.opts['environment']: continue - for match, data in body.items(): + for match, data in six.iteritems(body): if self.matcher.confirm_top( match, data, @@ -364,7 +365,7 @@ def top_matches(self, top): if saltenv not in matches: matches[saltenv] = [] for item in data: - if isinstance(item, string_types): + if isinstance(item, six.string_types): matches[saltenv].append(item) return matches @@ -421,7 +422,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): else: for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): - sub_sls, v = next(sub_sls.iteritems()) + sub_sls, v = next(six.iteritems(sub_sls)) defaults = v.get('defaults', {}) key = v.get('key', None) else: @@ -452,7 +453,7 @@ def render_pillar(self, matches): ''' pillar = {} errors = [] - for saltenv, pstates in matches.items(): + for saltenv, pstates in six.iteritems(matches): mods = set() for sls in pstates: pstate, mods, err = self.render_pstate(sls, saltenv, mods) @@ -520,7 +521,7 @@ def ext_pillar(self, pillar, pillar_dirs): if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} - for key, val in run.items(): + for key, val in six.iteritems(run): if key not in self.ext_pillars: err = ('Specified ext_pillar interface {0} is ' 'unavailable').format(key) From 50edc14d89d9833dda468ba75e0477cefd47467d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 7 Jan 2015 05:33:50 +0000 Subject: [PATCH 333/769] Py3 compatability --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d7efe27..b4c92f6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,7 +2,11 @@ ''' Salt package ''' + +# Import Python libs +from __future__ import absolute_import import warnings + # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( 'once', # Show once From 69051da853bdf16042055bd9e5d18771be666d0b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 21 Jan 2015 01:55:22 +0000 Subject: [PATCH 334/769] Py3 compatibility --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e22fb2a..30e9cb0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -312,7 +312,7 @@ def merge_tops(self, tops): if isinstance(comp, six.string_types): states[comp] = True if ignore_missing: - self.ignored_pillars[saltenv] = states.keys() + self.ignored_pillars[saltenv] = list(states.keys()) top[saltenv][tgt] = matches top[saltenv][tgt].extend(states) return self.sort_top_targets(top, orders) From 858c12493c165c1811d898fbe0b77b6311e3e828 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 2 Feb 2015 23:29:58 -0600 Subject: [PATCH 335/769] Accept pillar overrides in Pillar and RemotePillar This dictionary is passed through and made available to pillar and ext_pillar rendering functions. --- src/saltext/consul/pillar/__init__.py | 46 +++++++++++++++++++-------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b3691c6..de24658 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -5,10 +5,10 @@ from __future__ import absolute_import # Import python libs +import copy import os import collections import logging -from copy import copy # Import salt libs import salt.loader @@ -29,7 +29,7 @@ def merge_recurse(obj_a, obj_b): - copied = copy(obj_a) + copied = copy.copy(obj_a) return update(copied, obj_b) @@ -45,7 +45,8 @@ def merge_overwrite(obj_a, obj_b): return merge_recurse(obj_a, obj_b) -def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None): +def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, + pillar=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -57,24 +58,32 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None): ) # Backwards compatibility saltenv = env - - return { - 'remote': RemotePillar, - 'local': Pillar - }.get(opts['file_client'], Pillar)(opts, grains, id_, saltenv, ext, functions=funcs) + ptype = { + 'remote': RemotePillar, + 'local': Pillar + }.get(opts['file_client'], Pillar) + return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + pillar=pillar) class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): + def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + pillar=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains self.id_ = id_ self.channel = salt.transport.Channel.factory(opts) + self.pillar_override = {} + if pillar is not None: + if isinstance(pillar, dict): + self.pillar_override = pillar + else: + log.error('Pillar data must be a dictionary') def compile_pillar(self): ''' @@ -83,6 +92,7 @@ def compile_pillar(self): load = {'id': self.id_, 'grains': self.grains, 'saltenv': self.opts['environment'], + 'pillar_override': self.pillar_override, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -104,7 +114,8 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): + def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + pillar=None): # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client @@ -134,6 +145,13 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None): self.merge_strategy = opts['pillar_source_merging_strategy'] self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) + self.pillar_override = {} + if pillar is not None: + if isinstance(pillar, dict): + self.pillar_override = pillar + else: + log.error('Pillar data must be a dictionary') + def __valid_ext(self, ext): ''' @@ -435,7 +453,7 @@ def render_pillar(self, matches): Extract the sls pillar files from the matches and render them into the pillar ''' - pillar = {} + pillar = copy.copy(self.pillar_override) errors = [] for saltenv, pstates in matches.items(): mods = set() @@ -501,6 +519,8 @@ def ext_pillar(self, pillar, pillar_dirs): log.critical('The "ext_pillar" option is malformed') return pillar ext = None + # Bring in CLI pillar data + pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') @@ -569,7 +589,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): ''' Render the pillar data and return ''' - top, terrors = self.get_top() + top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) @@ -583,7 +603,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - errors.extend(terrors) + errors.extend(top_errors) if self.opts.get('pillar_opts', True): mopts = dict(self.opts) if 'grains' in mopts: From e30be1ee8c718558520fc9eea32c1a59f578708f Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 3 Feb 2015 06:08:50 -0700 Subject: [PATCH 336/769] lint fix --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index de24658..c27bbaf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -152,7 +152,6 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, else: log.error('Pillar data must be a dictionary') - def __valid_ext(self, ext): ''' Check to see if the on demand external pillar is allowed From 45bbf3fcef3eef5399584abb8ad7ec245759e116 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Tue, 3 Feb 2015 14:49:30 -0700 Subject: [PATCH 337/769] Don't try to compile empty pillars Fixes bug where empty template warning if no pillar exists --- src/saltext/consul/pillar/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 28cec5b..6bb616e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -217,17 +217,19 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): - tops[saltenv].append( - compile_template( - self.client.cache_file( - self.opts['state_top'], - saltenv - ), - self.rend, - self.opts['renderer'], - saltenv=saltenv - ) + top = self.client.cache_file( + self.opts['state_top'], + saltenv ) + if top: + tops[saltenv].append( + compile_template( + top, + self.rend, + self.opts['renderer'], + saltenv=saltenv + ) + ) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}' From 7eb5862149b8f9f20cf24b5cb577d67394b86828 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 3 Feb 2015 19:47:01 -0600 Subject: [PATCH 338/769] Move Pillar's merge_sources to salt.utils.dictupdate --- src/saltext/consul/pillar/__init__.py | 54 ++++----------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c27bbaf..b51b149 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -18,8 +18,7 @@ import salt.transport from salt.ext.six import string_types from salt.template import compile_template -from salt.utils.dictupdate import update -from salt.utils.serializers.yamlex import merge_recursive +from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict from salt.version import __version__ import salt.ext.six as six @@ -28,23 +27,6 @@ log = logging.getLogger(__name__) -def merge_recurse(obj_a, obj_b): - copied = copy.copy(obj_a) - return update(copied, obj_b) - - -def merge_aggregate(obj_a, obj_b): - return merge_recursive(obj_a, obj_b, level=1) - - -def merge_overwrite(obj_a, obj_b): - for obj in obj_b: - if obj in obj_a: - obj_a[obj] = obj_b[obj] - return obj_a - return merge_recurse(obj_a, obj_b) - - def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, pillar=None): ''' @@ -441,7 +423,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key: nstate } - state = self.merge_sources(state, nstate) + state = merge(state, nstate, + self.merge_strategy) if err: errors += err @@ -474,7 +457,7 @@ def render_pillar(self, matches): ) ) continue - pillar = self.merge_sources(pillar, pstate) + pillar = merge(pillar, pstate, self.merge_strategy) return pillar, errors @@ -556,34 +539,10 @@ def ext_pillar(self, pillar, pillar_dirs): ) ) if ext: - pillar = self.merge_sources(pillar, ext) + pillar = merge(pillar, ext, self.merge_strategy) ext = None return pillar - def merge_sources(self, obj_a, obj_b): - strategy = self.merge_strategy - - if strategy == 'smart': - renderer = self.opts.get('renderer', 'yaml') - if renderer == 'yamlex' or renderer.startswith('yamlex_'): - strategy = 'aggregate' - else: - strategy = 'recurse' - - if strategy == 'recurse': - merged = merge_recurse(obj_a, obj_b) - elif strategy == 'aggregate': - #: level = 1 merge at least root data - merged = merge_aggregate(obj_a, obj_b) - elif strategy == 'overwrite': - merged = merge_overwrite(obj_a, obj_b) - else: - log.warning('unknown merging strategy {0}, ' - 'fallback to recurse'.format(strategy)) - merged = merge_recurse(obj_a, obj_b) - - return merged - def compile_pillar(self, ext=True, pillar_dirs=None): ''' Render the pillar data and return @@ -594,7 +553,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = self.merge_sources(pillar, self.opts['pillar']) + pillar = merge(pillar, self.opts['pillar'], + self.merge_strategy) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 40f66a698e2709a323788ed5065b19cf13fa3e0f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 3 Feb 2015 19:47:58 -0600 Subject: [PATCH 339/769] Update ref to salt.utils.dictupdate.merge() --- src/saltext/consul/pillar/__init__.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b51b149..5624585 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -423,8 +423,11 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key: nstate } - state = merge(state, nstate, - self.merge_strategy) + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml')) if err: errors += err @@ -457,7 +460,11 @@ def render_pillar(self, matches): ) ) continue - pillar = merge(pillar, pstate, self.merge_strategy) + pillar = merge( + pillar, + pstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml')) return pillar, errors @@ -539,7 +546,11 @@ def ext_pillar(self, pillar, pillar_dirs): ) ) if ext: - pillar = merge(pillar, ext, self.merge_strategy) + pillar = merge( + pillar, + ext, + self.merge_strategy, + self.opts.get('renderer', 'yaml')) ext = None return pillar @@ -553,8 +564,10 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = merge(pillar, self.opts['pillar'], - self.merge_strategy) + pillar = merge(pillar, + self.opts['pillar'], + self.merge_strategy, + self.opts.get('renderer', 'yaml')) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 3af7c55e45a52e9945efe191e4133c08045a0ba9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 23 Feb 2015 12:42:52 +0000 Subject: [PATCH 340/769] Add system encoding detection Try to detect the system encoding as soon as possible. `sys.stdin.encoding` is the most reliable source but it's reset to `None` once a process is daemonized. If it's none, follow additional paths to retrieve this information. --- src/saltext/consul/__init__.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d7efe27..31a9ae6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -24,3 +24,41 @@ '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', UserWarning ) + + +def __define_global_system_encoding_variable__(): + import sys + # This is the most trustworthy source of the system encoding, though, if + # salt is being imported after being daemonized, this information is lost + # and reset to None + encoding = sys.stdin.encoding + if not encoding: + # If the system is properly codfigured this should return a valid + # encoding. MS Windows has problems with this and reports the wrong + # encoding + import locale + encoding = locale.getdefaultlocale()[-1] + + # This is now garbage collectable + del locale + if not encoding: + # This is most likely asccii which is not the best but we were + # unable to find a better encoding + encoding = sys.getdefaultencoding() + + # Import 3rd-party libs + import salt.ext.six.moves.builtins as builtins # pylint: disable=import-error,no-name-in-module + + # Define the detected encoding as a built-in variable for ease of use + setattr(builtins, '__salt_system_encoding__', encoding) + + # This is now garbage collectable + del sys + del builtins + del encoding + + +__define_global_system_encoding_variable__() + +# This is now garbage collectable +del __define_global_system_encoding_variable__ From a26f4567f33fedf9f1caac42be61918fc052e9f9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 26 Feb 2015 00:55:21 +0000 Subject: [PATCH 341/769] We can't use 'six.moves.builtins`. Apparently the `six` `builtins` get deleted sooner than we hoped for. Every builtin gets reset to `None`. Fixes #21036 --- src/saltext/consul/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 31a9ae6..420f10d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -46,8 +46,13 @@ def __define_global_system_encoding_variable__(): # unable to find a better encoding encoding = sys.getdefaultencoding() - # Import 3rd-party libs - import salt.ext.six.moves.builtins as builtins # pylint: disable=import-error,no-name-in-module + # We can't use six.moves.builtins because these builtins get deleted sooner + # than expected. See: + # https://github.com/saltstack/salt/issues/21036 + if sys.version_info.major < 3: + import __builtin__ as builtins + else: + import builtins # pylint: disable=import-error # Define the detected encoding as a built-in variable for ease of use setattr(builtins, '__salt_system_encoding__', encoding) From 5d4c0f7f7ec86d0b250201d090f03983eba04df2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 26 Feb 2015 09:49:36 +0000 Subject: [PATCH 342/769] Correct indentation. --- src/saltext/consul/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 420f10d..1d7559a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,27 +2,29 @@ ''' Salt package ''' +# Import Python Libs import warnings + # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( - 'once', # Show once - '', # No deprecation message match - DeprecationWarning, # This filter is for DeprecationWarnings - r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' + 'once', # Show once + '', # No deprecation message match + DeprecationWarning, # This filter is for DeprecationWarnings + r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' ) # While we are supporting Python2.6, hide nested with-statements warnings warnings.filterwarnings( - 'ignore', - 'With-statements now directly support multiple context managers', - DeprecationWarning + 'ignore', + 'With-statements now directly support multiple context managers', + DeprecationWarning ) # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( - 'ignore', - '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', - UserWarning + 'ignore', + '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + UserWarning ) From 2a4db7e79bed2b3da3a781bd7eacb92b80ce0a83 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 26 Feb 2015 10:53:20 -0700 Subject: [PATCH 343/769] Make sys.version_info python 2.6 compatible Refs #21053: Named component attributes weren't added until 2.7 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 420f10d..92949c3 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -49,7 +49,7 @@ def __define_global_system_encoding_variable__(): # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: # https://github.com/saltstack/salt/issues/21036 - if sys.version_info.major < 3: + if sys.version_info[0] < 3: import __builtin__ as builtins else: import builtins # pylint: disable=import-error From 4f283a33f5d9463399a1328f6362598c31c629e4 Mon Sep 17 00:00:00 2001 From: s8weber Date: Tue, 3 Mar 2015 20:53:36 -0500 Subject: [PATCH 344/769] add pillar_safe_render_error config allow a salt admin to config if the detailed render error of pillars can be passed to the minion. --- src/saltext/consul/pillar/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5624585..8eaf280 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -387,7 +387,10 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): sls, exc ) log.critical(msg) - errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) + if self.opts.get('pillar_safe_render_error', True): + errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) + else: + errors.append(msg) mods.add(sls) nstate = None if state: From 507d3505a548b31464bcf37fa52da99ef7c3df4e Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Mar 2015 11:31:52 -0600 Subject: [PATCH 345/769] Latest Python 3 compat fixes, plus a versionadded tag --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d4a5c13..80e0dd1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -54,7 +54,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins + import __builtin__ as builtins # pylint: disable=py3-compat else: import builtins # pylint: disable=import-error From 5e58ecf2bf6d5379840ee7acdbcb72c0d7626090 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Mar 2015 12:49:32 -0600 Subject: [PATCH 346/769] Better pylint disable --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 80e0dd1..3e78598 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -54,7 +54,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins # pylint: disable=py3-compat + import __builtin__ as builtins # pylint: disable=disable-py3-compat else: import builtins # pylint: disable=import-error From 05e660587cd5ebc16992019e93d2b9b330f94847 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 9 Mar 2015 13:40:37 -0600 Subject: [PATCH 347/769] It's incompatible-py3-code --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3e78598..dc5bae4 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -54,7 +54,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins # pylint: disable=disable-py3-compat + import __builtin__ as builtins # pylint: disable=incompatible-py3-code else: import builtins # pylint: disable=import-error From c15c0ef11367d4c28e5934eb31960e6a2d9f75f0 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Wed, 1 Apr 2015 15:10:56 +0000 Subject: [PATCH 348/769] make __utils__ available to execution modules --- src/saltext/consul/pillar/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 25e17f8..00b517b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -110,10 +110,11 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, # if we didn't pass in functions, lets load them if functions is None: + utils = salt.loader.utils(opts) if opts.get('file_client', '') == 'local': - self.functions = salt.loader.minion_mods(opts) + self.functions = salt.loader.minion_mods(opts, utils=utils) else: - self.functions = salt.loader.minion_mods(self.opts) + self.functions = salt.loader.minion_mods(self.opts, utils=utils) else: self.functions = functions From 7977fd355bcc3f69d9d0e91bacf49759d5ab4596 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Tue, 14 Apr 2015 09:21:51 -0700 Subject: [PATCH 349/769] Add AsyncPillar interfaces --- src/saltext/consul/pillar/__init__.py | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 00b517b..bc93671 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -16,6 +16,7 @@ import salt.minion import salt.crypt import salt.transport +from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict @@ -24,6 +25,7 @@ # Import 3rd-party libs import salt.ext.six as six +import tornado.gen log = logging.getLogger(__name__) @@ -49,6 +51,74 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, pillar=pillar) +# TODO: migrate everyone to this one! +def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, + pillar=None): + ''' + Return the correct pillar driver based on the file_client option + ''' + if env is not None: + salt.utils.warn_until( + 'Boron', + 'Passing a salt environment should be done using \'saltenv\' ' + 'not \'env\'. This functionality will be removed in Salt Boron.' + ) + # Backwards compatibility + saltenv = env + ptype = { + 'remote': AsyncRemotePillar, + #'local': AsyncPillar # TODO: implement + }.get(opts['file_client'], Pillar) + return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + pillar=pillar) + + +class AsyncRemotePillar(object): + ''' + Get the pillar from the master + ''' + def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + pillar=None): + self.opts = opts + self.opts['environment'] = saltenv + self.ext = ext + self.grains = grains + self.id_ = id_ + self.channel = salt.transport.client.AsyncReqChannel.factory(opts) + self.pillar_override = {} + if pillar is not None: + if isinstance(pillar, dict): + self.pillar_override = pillar + else: + log.error('Pillar data must be a dictionary') + + @tornado.gen.coroutine + def compile_pillar(self): + ''' + Return a future which will contain the pillar data from the master + ''' + load = {'id': self.id_, + 'grains': self.grains, + 'saltenv': self.opts['environment'], + 'pillar_override': self.pillar_override, + 'ver': '2', + 'cmd': '_pillar'} + if self.ext: + load['ext'] = self.ext + ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( + load, + dictkey='pillar', + ) + + if not isinstance(ret_pillar, dict): + msg = ('Got a bad pillar from master, type {0}, expecting dict: ' + '{1}').format(type(ret_pillar).__name__, ret_pillar) + log.error(msg) + # raise an exception! Pillar isn't empty, we can't sync it! + raise SaltClientError(msg) + raise tornado.gen.Return(ret_pillar) + + class RemotePillar(object): ''' Get the pillar from the master From 9bda2c5a58d474861ece3eb3ef31818454ec5188 Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Sat, 2 May 2015 10:37:36 -0600 Subject: [PATCH 350/769] create and parse salt:// URLs with url util --- src/saltext/consul/pillar/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bc93671..a07b9c9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -16,6 +16,7 @@ import salt.minion import salt.crypt import salt.transport +import salt.utils.url from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -244,9 +245,9 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): - opts['state_top'] = os.path.join('salt://', opts['state_top'][1:]) + opts['state_top'] = salt.utils.url.create(opts['state_top'][1:]) else: - opts['state_top'] = os.path.join('salt://', opts['state_top']) + opts['state_top'] = salt.utils.url.create(opts['state_top']) if self.__valid_ext(ext): if 'ext_pillar' in opts: opts['ext_pillar'].append(ext) From d053b0d835344548724c3538f11a20b268e0fc55 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 7 May 2015 18:59:55 -0700 Subject: [PATCH 351/769] Initial commit of a execution module to interact with and manage Consul --- src/saltext/consul/modules/consul.py | 2408 ++++++++++++++++++++++++++ 1 file changed, 2408 insertions(+) create mode 100644 src/saltext/consul/modules/consul.py diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py new file mode 100644 index 0000000..db72c10 --- /dev/null +++ b/src/saltext/consul/modules/consul.py @@ -0,0 +1,2408 @@ +# -*- coding: utf-8 -*- +''' +Interact with Consul + +https://www.consul.io + +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import 3rd-party libs +# pylint: disable=import-error,no-name-in-module,redefined-builtin +from salt.ext.six.moves.urllib.parse import urljoin as _urljoin +import salt.ext.six.moves.http_client +# pylint: enable=import-error,no-name-in-module + +import base64 + +try: + import requests + from requests.exceptions import ConnectionError + ENABLED = True +except ImportError: + ENABLED = False + +import logging +log = logging.getLogger(__name__) + +__virtualname__ = 'consul' + + +def _query(function, + consul_url, + api_version='v1', + method='GET', + data=None, + query_params=None): + ''' + Consul object method function to construct and execute on the API URL. + + :param api_url: The Consul api url. + :param api_version The Consul api version + :param function: The Consul api function to perform. + :param method: The HTTP method, e.g. GET or POST. + :param data: The data to be sent for POST method. + :return: The json response from the API call or False. + ''' + headers = {} + if not query_params: + query_params = {} + + if data is None: + data = {} + + ret = {'data': '', + 'res': True} + + base_url = _urljoin(consul_url, '{0}/'.format(api_version)) + url = _urljoin(base_url, function, False) + + try: + result = requests.request( + method=method, + url=url, + headers=headers, + params=query_params, + data=data, + verify=True, + ) + except ConnectionError as e: + ret['data'] = e + ret['res'] = False + return ret + + if result.status_code == salt.ext.six.moves.http_client.OK: + result = result.json() + if result: + ret['data'] = result + ret['res'] = True + else: + ret['res'] = False + elif result.status_code == salt.ext.six.moves.http_client.NO_CONTENT: + ret['res'] = False + elif result.status_code == salt.ext.six.moves.http_client.NOT_FOUND: + ret['data'] = 'Key not found.' + ret['res'] = False + else: + result = result.json() + if result: + ret['data'] = result + ret['res'] = True + else: + ret['res'] = False + return ret + + +def list(consul_url, key=None, **kwargs): + ''' + List keys in Consul + + :param consul_url: The Consul server URL. + :param key: The key to use as the starting point for the list. + :return: The list of keys. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.list + + salt '*' consul.list key='web' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + if 'recurse' in kwargs: + query_params['recurse'] = 'True' + + # No key so recurse and show all values + if not key: + query_params['recurse'] = 'True' + function = 'kv/' + else: + function = 'kv/{0}'.format(key) + + query_params['keys'] = 'True' + + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def get(consul_url, key, recurse=False, decode=False, raw=False): + ''' + Get key from Consul + + :param consul_url: The Consul server URL. + :param key: The key to use as the starting point for the list. + :param recurse: Return values recursively beginning at the value of key. + :param decode: By default values are stored as Base64 encoded values, + decode will return the whole key with the value decoded. + :param raw: Simply return the decoded value of the key. + :return: The keys in Consul. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.get key='web/key1' + + salt '*' consul.list key='web' recurse='True + + salt '*' consul.list key='web' recurse='True' decode='True' + + By default values stored in Consul are base64 encoded, passing the + decode option will show them as the decoded values. + + salt '*' consul.list key='web' recurse='True' decode='True' raw='True' + + By default Consult will return other information about the key, the raw + option will return only the raw value. + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + function = 'kv/{0}'.format(key) + if recurse: + query_params['recurse'] = 'True' + if raw: + query_params['raw'] = True + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + + if ret['res']: + if decode: + for item in ret['data']: + item['Value'] = base64.b64decode(item['Value']) + return ret + + +def put(consul_url, key, value, **kwargs): + ''' + Put values into Consul + + :param consul_url: The Consul server URL. + :param key: The key to use as the starting point for the list. + :param value: The value to set the key to. + :param flags: This can be used to specify an unsigned value + between 0 and 2^64-1. Clients can choose to use + this however makes sense for their application. + :param cas: This flag is used to turn the PUT into a + Check-And-Set operation. + :param acquire: This flag is used to turn the PUT into a + lock acquisition operation. + :param release: This flag is used to turn the PUT into a + lock release operation. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.put key='web/key1' value="Hello there" + + salt '*' consul.put key='web/key1' value="Hello there" + acquire='d5d371f4-c380-5280-12fd-8810be175592' + + salt '*' consul.put key='web/key1' value="Hello there" + release='d5d371f4-c380-5280-12fd-8810be175592' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + available_sessions = session_list(consul_url=consul_url, return_list=True) + _current = get(consul_url=consul_url, key=key) + + if 'flags' in kwargs: + if not kwargs['flags'] >= 0 and not kwargs['flags'] <= 2**64: + query_params['flags'] = kwargs['flags'] + + if 'cas' in kwargs: + if _current['res']: + if kwargs['cas'] == 0: + ret['message'] = ('Key {0} exists, index ' + 'must be non-zero.'.format(key)) + ret['res'] = False + return ret + + if kwargs['cas'] != _current['data']['ModifyIndex']: + ret['message'] = ('Key {0} exists, but indexes ' + 'do not match.'.format(key)) + ret['res'] = False + return ret + query_params['cas'] = kwargs['cas'] + else: + ret['message'] = ('Key {0} does not exists, ' + 'CAS argument can not be used.'.format(key)) + ret['res'] = False + return ret + else: + log.error('Key {0} does not exist. Skipping release.') + + if 'acquire' in kwargs: + if kwargs['acquire'] not in available_sessions: + ret['message'] = ('{0} is not a valid ', + 'session.'.format(kwargs['acquire'])) + ret['res'] = False + return ret + + query_params['acquire'] = kwargs['acquire'] + + if 'release' in kwargs: + if _current['res']: + if 'Session' in _current['data']: + if _current['data']['Session'] == kwargs['release']: + query_params['release'] = kwargs['release'] + else: + ret['message'] = ('{0} locked by ', + 'another session.'.format(key)) + ret['res'] = False + return ret + + else: + ret['message'] = ('{0} is not a valid ', + 'session.'.format(kwargs['acquire'])) + ret['res'] = False + else: + log.error('Key {0} does not exist. Skipping release.') + + data = value + function = 'kv/{0}'.format(key) + ret = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data, + query_params=query_params) + if ret['res']: + ret['res'] = True + ret['data'] = 'Added key {0} with value {1}.'.format(key, value) + else: + ret['res'] = False + ret['data'] = 'Unable to add key {0} with value {1}.'.format(key, value) + return ret + + +def delete(consul_url, key, **kwargs): + ''' + Delete values from Consul + + :param consul_url: The Consul server URL. + :param key: The key to use as the starting point for the list. + :param recurse: Delete values recursively beginning at the value of key. + :param cas: This flag is used to turn the DELETE into + a Check-And-Set operation. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.delete key='web' + + salt '*' consul.delete key='web' recurse='True' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + if 'recurse' in kwargs: + query_params['recurse'] = True + + if 'cas' in kwargs: + if kwargs['cas'] > 0: + query_params['cas'] = kwargs['cas'] + else: + ret['message'] = ('Check and Set Operation ', + 'value must be greater than 0.') + ret['res'] = False + return ret + + function = 'kv/{0}'.format(key) + ret = _query(consul_url=consul_url, + function=function, + method='DELETE', + query_params=query_params) + + if ret['res']: + ret['res'] = True + ret['message'] = 'Deleted key {0}.'.format(key) + else: + ret['res'] = False + ret['message'] = 'Unable to delete key {0}.'.format(key) + return ret + + +def agent_checks(consul_url): + ''' + Returns the checks the local agent is managing + + :param consul_url: The Consul server URL. + :return: Returns the checks the local agent is managing + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_checks + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/checks' + ret = _query(consul_url=consul_url, + function=function, + method='GET') + return ret + + +def agent_services(consul_url): + ''' + Returns the services the local agent is managing + + :param consul_url: The Consul server URL. + :return: Returns the services the local agent is managing + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_services + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/services' + ret = _query(consul_url=consul_url, + function=function, + method='GET') + return ret + + +def agent_members(consul_url, **kwargs): + ''' + Returns the members as seen by the local serf agent + + :param consul_url: The Consul server URL. + :return: Returns the members as seen by the local serf agent + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_members + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'wan' in kwargs: + query_params['wan'] = kwargs['wan'] + + function = 'agent/members' + ret = _query(consul_url=consul_url, + function=function, + method='GET', + query_params=query_params) + return ret + + +def agent_self(consul_url): + ''' + Returns the local node configuration + + :param consul_url: The Consul server URL. + :return: Returns the local node configuration + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_self + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/self' + ret = _query(consul_url=consul_url, + function=function, + method='GET', + query_params=query_params) + return ret + + +def agent_maintenance(consul_url, **kwargs): + ''' + Manages node maintenance mode + + :param consul_url: The Consul server URL. + :param enable: The enable flag is required. + Acceptable values are either true + (to enter maintenance mode) or + false (to resume normal operation). + :param reason: If provided, its value should be a + text string explaining the reason for + placing the node into maintenance mode. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_maintenance enable='False' reason='Upgrade in progress' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'enable' in kwargs: + query_params['enable'] = kwargs['enable'] + else: + ret['message'] = 'Required parameter "enable" is missing.' + ret['res'] = False + return ret + + if 'reason' in kwargs: + query_params['reason'] = kwargs['reason'] + + function = 'agent/maintenance' + res = _query(consul_url=consul_url, + function=function, + method='GET', + query_params=query_params) + if res['res']: + ret['res'] = True + ret['message'] = ('Agent maintenance mode ' + '{0}ed.'.format(kwargs['enable'])) + else: + ret['res'] = True + ret['message'] = 'Unable to change maintenance mode for agent.' + return ret + + +def agent_join(consul_url, address, **kwargs): + ''' + Triggers the local agent to join a node + + :param consul_url: The Consul server URL. + :param address: The address for the agent to connect to. + :param wan: Causes the agent to attempt to join using the WAN pool. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_join address='192.168.1.1' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'wan' in kwargs: + query_params['wan'] = kwargs['wan'] + + function = 'agent/join/{0}'.format(address) + res = _query(consul_url=consul_url, + function=function, + method='GET', + query_params=query_params) + if res['res']: + ret['res'] = True + ret['message'] = ('Agent maintenance mode ' + '{0}ed.'.format(kwargs['enable'])) + else: + ret['res'] = False + ret['message'] = 'Unable to change maintenance mode for agent.' + return ret + + +def agent_leave(consul_url, node): + ''' + Used to instruct the agent to force a node into the left state. + + :param consul_url: The Consul server URL. + :param node: The node the agent will force into left state + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_leave node='web1.example.com' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/force-leave/{0}'.format(node) + res = _query(consul_url=consul_url, + function=function, + method='GET', + query_params=query_params) + if res['res']: + ret['res'] = True + ret['message'] = 'Node {0} put in leave state.'.format(node) + else: + ret['res'] = False + ret['message'] = 'Unable to change state for {0}.'.format(node) + return ret + + +def agent_check_register(consul_url, **kwargs): + ''' + The register endpoint is used to add a new check to the local agent. + + :param consul_url: The Consul server URL. + :param name: The description of what the check is for. + :param id: The unique name to use for the check, if not + provided 'name' is used. + :param notes: Human readable description of the check. + :param script: If script is provided, the check type is + a script, and Consul will evaluate that script + based on the interval parameter. + :param http: Check will perform an HTTP GET request against + the value of HTTP (expected to be a URL) based + on the interval parameter. + :param ttl: If a TTL type is used, then the TTL update endpoint + must be used periodically to update the state of the check. + :param interval: Interval at which the check should run. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_check_register name='Memory Utilization' + script='/usr/local/bin/check_mem.py' interval='15s' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'name' in kwargs: + data['Name'] = kwargs['name'] + else: + ret['message'] = 'Required parameter "name" is missing.' + ret['res'] = False + return ret + + if True not in [True for item in ('script', 'http') if item in kwargs]: + ret['message'] = 'Required parameter "script" or "http" is missing.' + ret['res'] = False + return ret + + if 'id' in kwargs: + data['ID'] = kwargs['id'] + + if 'notes' in kwargs: + data['Notes'] = kwargs['notes'] + + if 'script' in kwargs: + if 'interval' not in kwargs: + ret['message'] = 'Required parameter "interval" is missing.' + ret['res'] = False + return ret + data['Script'] = kwargs['script'] + data['Interval'] = kwargs['interval'] + + if 'http' in kwargs: + if 'interval' not in kwargs: + ret['message'] = 'Required parameter "interval" is missing.' + ret['res'] = False + return ret + data['HTTP'] = kwargs['http'] + data['Interval'] = kwargs['interval'] + + function = 'agent/check/register' + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + if res['res']: + ret['res'] = True + ret['message'] = ('Check {0} added to agent.'.format(kwargs['name'])) + else: + ret['res'] = False + ret['message'] = 'Unable to add check to agent.' + return ret + + +def agent_check_deregister(consul_url, checkid): + ''' + The agent will take care of deregistering the check from the Catalog. + + :param consul_url: The Consul server URL. + :param checkid: The ID of the check to deregister from Consul. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_check_register name='Memory Utilization' + script='/usr/local/bin/check_mem.py' interval='15s' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/check/deregister/{0}'.format(checkid) + res = _query(consul_url=consul_url, + function=function, + method='GET') + if res['res']: + ret['res'] = True + ret['message'] = ('Check {0} removed from agent.'.format(checkid)) + else: + ret['res'] = False + ret['message'] = 'Unable to remove check from agent.' + return ret + + +def agent_check_pass(consul_url, checkid, **kwargs): + ''' + This endpoint is used with a check that is of the TTL type. When this + is called, the status of the check is set to passing and the TTL + clock is reset. + + :param consul_url: The Consul server URL. + :param checkid: The ID of the check to mark as passing. + :param note: A human-readable message with the status of the check. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_check_pass checkid='redis_check1' + note='Forcing check into passing state.' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'note' in kwargs: + query_params['note'] = kwargs['note'] + + function = 'agent/check/pass/{0}'.format(checkid) + res = _query(consul_url=consul_url, + function=function, + query_params=query_params, + method='GET') + if res['res']: + ret['res'] = True + ret['message'] = 'Check {0} marked as passing.'.format(checkid) + else: + ret['res'] = False + ret['message'] = 'Unable to update check {0}.'.format(checkid) + return ret + + +def agent_check_warn(consul_url, checkid, **kwargs): + ''' + This endpoint is used with a check that is of the TTL type. When this + is called, the status of the check is set to warning and the TTL + clock is reset. + + :param consul_url: The Consul server URL. + :param checkid: The ID of the check to deregister from Consul. + :param note: A human-readable message with the status of the check. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_check_warn checkid='redis_check1' + note='Forcing check into warning state.' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'note' in kwargs: + query_params['note'] = kwargs['note'] + + function = 'agent/check/warn/{0}'.format(checkid) + res = _query(consul_url=consul_url, + function=function, + query_params=query_params, + method='GET') + if res['res']: + ret['res'] = True + ret['message'] = 'Check {0} marked as warning.'.format(checkid) + else: + ret['res'] = False + ret['message'] = 'Unable to update check {0}.'.format(checkid) + return ret + + +def agent_check_fail(consul_url, checkid, **kwargs): + ''' + This endpoint is used with a check that is of the TTL type. When this + is called, the status of the check is set to critical and the + TTL clock is reset. + + :param consul_url: The Consul server URL. + :param checkid: The ID of the check to deregister from Consul. + :param note: A human-readable message with the status of the check. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_check_fail checkid='redis_check1' + note='Forcing check into critical state.' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'note' in kwargs: + query_params['note'] = kwargs['note'] + + function = 'agent/check/fail/{0}'.format(checkid) + res = _query(consul_url=consul_url, + function=function, + query_params=query_params, + method='GET') + if res['res']: + ret['res'] = True + ret['message'] = 'Check {0} marked as critical.'.format(checkid) + else: + ret['res'] = False + ret['message'] = 'Unable to update check {0}.'.format(checkid) + return ret + + +def agent_service_register(consul_url, **kwargs): + ''' + The used to add a new service, with an optional + health check, to the local agent. + + :param consul_url: The Consul server URL. + :param name: A name describing the service. + :param address: The address used by the service, defaults + to the address of the agent. + :param port: The port used by the service. + :param id: Unique ID to identify the service, if not + provided the value of the name parameter is used. + :param tags: Identifying tags for service, string or list. + :param script: If script is provided, the check type is + a script, and Consul will evaluate that script + based on the interval parameter. + :param http: Check will perform an HTTP GET request against + the value of HTTP (expected to be a URL) based + on the interval parameter. + :param check_ttl: If a TTL type is used, then the TTL update + endpoint must be used periodically to update + the state of the check. + :param check_interval: Interval at which the check should run. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_service_register name='redis' + tags='["master", "v1"]' address="127.0.0.1" port="8080" + check_script="/usr/local/bin/check_redis.py" interval="10s" + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'name' in kwargs: + data['Name'] = kwargs['name'] + + if 'address' in kwargs: + data['Address'] = kwargs['address'] + + if 'port' in kwargs: + data['Port'] = kwargs['port'] + + if 'id' in kwargs: + data['ID'] = kwargs['id'] + + if 'tags' in kwargs: + _tags = kwargs['tags'] + if not isinstance(_tags, list): + _tags = [_tags] + data['Tags'] = _tags + + check_elements = ('check_script', 'check_http', 'check_ttl') + if True in [True for item in check_elements if item in kwargs]: + data['Check'] = {} + + if 'check_script' in kwargs: + if 'interval' not in kwargs: + ret['message'] = 'Required parameter "interval" is missing.' + ret['res'] = False + return ret + data['Check']['Script'] = kwargs['check_script'] + data['Check']['Interval'] = kwargs['check_interval'] + + if 'check_ttl' in kwargs: + data['Check']['TTL'] = kwargs['check_ttl'] + + if 'check_http' in kwargs: + if 'interval' not in kwargs: + ret['message'] = 'Required parameter "interval" is missing.' + ret['res'] = False + return ret + data['Check']['HTTP'] = kwargs['check_http'] + data['Check']['Interval'] = kwargs['check_interval'] + + function = 'agent/service/register' + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + if res['res']: + ret['res'] = True + ret['message'] = 'Service {0} registered on agent.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = 'Unable to register service {0}.'.format(kwargs['name']) + return ret + + +def agent_service_deregister(consul_url, serviceid): + ''' + Used to remove a service. + + :param consul_url: The Consul server URL. + :param name: A name describing the service. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_service_deregister serviceid='redis' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'agent/service/deregister/{0}'.format(serviceid) + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + if res['res']: + ret['res'] = True + ret['message'] = 'Service {0} removed from agent.'.format(serviceid) + else: + ret['res'] = False + ret['message'] = 'Unable to remove service {0}.'.format(serviceid) + return ret + + +def agent_service_maintenance(consul_url, serviceid, **kwargs): + ''' + Used to place a service into maintenance mode. + + :param consul_url: The Consul server URL. + :param serviceid: A name of the service. + :param enable: Whether the service should be enabled or disabled. + :param reason: A human readable message of why the service was + enabled or disabled. + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.agent_service_deregister serviceid='redis' + enable='True' reason='Down for upgrade' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'enable' in kwargs: + query_params['enable'] = kwargs['enable'] + else: + ret['message'] = 'Required parameter "enable" is missing.' + ret['res'] = False + return ret + + if 'reason' in kwargs: + query_params['reason'] = kwargs['reason'] + + function = 'agent/service/maintenance/{0}'.format(serviceid) + res = _query(consul_url=consul_url, + function=function, + query_params=query_params) + + if res['res']: + ret['res'] = True + ret['message'] = ('Service {0} set in ' + 'maintenance mode.'.format(serviceid)) + else: + ret['res'] = False + ret['message'] = ('Unable to set service ' + '{0} to maintenance mode.'.format(serviceid)) + return ret + + +def session_create(consul_url, **kwargs): + ''' + Used to create a session. + + :param consul_url: The Consul server URL. + :param lockdelay: Duration string using a "s" suffix for seconds. + The default is 15s. + :param node: Must refer to a node that is already registered, + if specified. By default, the agent's own node + name is used. + :param name: A human-readable name for the session + :param checks: A list of associated health checks. It is highly + recommended that, if you override this list, you + include the default "serfHealth". + :param behavior: Can be set to either release or delete. This controls + the behavior when a session is invalidated. By default, + this is release, causing any locks that are held to be + released. Changing this to delete causes any locks that + are held to be deleted. delete is useful for creating + ephemeral key/value entries. + :param ttl: Session is invalidated if it is not renewed before + the TTL expires + :return: Boolean and message indicating success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.session_create node='node1' name='my-session' + behavior='delete' ttl='3600s' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + data = {} + + if 'lockdelay' in kwargs: + data['LockDelay'] = kwargs['lockdelay'] + + if 'node' in kwargs: + data['Node'] = kwargs['node'] + + if 'name' in kwargs: + data['Name'] = kwargs['name'] + + if 'checks' in kwargs: + data['Touch'] = kwargs['touch'] + + if 'behavior' in kwargs: + if not kwargs['behavior'] in ('delete', 'release'): + ret['message'] = ('Behavior must be ', + 'either delete or release.') + ret['res'] = False + return ret + data['Behavior'] = kwargs['behavior'] + + if 'ttl' in kwargs: + _ttl = kwargs['ttl'] + if str(_ttl).endswith('s'): + _ttl = _ttl[:-1] + + if not int(_ttl) >= 0 and not int(_ttl) <= 3600: + ret['message'] = ('TTL must be ', + 'between 0 and 3600.') + ret['res'] = False + return ret + data['TTL'] = '{0}s'.format(_ttl) + + function = 'session/create' + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + + if res['res']: + ret['res'] = True + ret['message'] = 'Created session {0}.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = 'Unable to create session {0}.'.format(kwargs['name']) + return ret + + +def session_list(consul_url, return_list=False, **kwargs): + ''' + Used to list sessions. + + :param consul_url: The Consul server URL. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param return_list: By default, all information about the sessions is + returned, using the return_list parameter will return + a list of session IDs. + :return: A list of all available sessions. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.session_list + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'session/list' + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + + if return_list: + _list = [] + for item in ret['data']: + _list.append(item['ID']) + return _list + return ret + + +def session_destroy(consul_url, session, **kwargs): + ''' + Destroy session + + :param consul_url: The Consul server URL. + :param session: The ID of the session to destroy. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.session_destroy session='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'session/destroy/{0}'.format(session) + res = _query(consul_url=consul_url, + function=function, + query_params=query_params) + if res['res']: + ret['res'] = True + ret['message'] = 'Created Service {0}.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = 'Unable to create service {0}.'.format(kwargs['name']) + return ret + + +def session_info(consul_url, session, **kwargs): + ''' + Information about a session + + :param consul_url: The Consul server URL. + :param session: The ID of the session to return information about. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.session_info session='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + query_params = {} + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'session/info/{0}'.format(session) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def catalog_register(consul_url, **kwargs): + ''' + Registers a new node, service, or check + + :param consul_url: The Consul server URL. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param node: The node to register. + :param address: The address of the node. + :param service: The service that will be registered. + :param service_address: The address that the service listens on. + :param service_port: The port for the service. + :param service_id: A unique identifier for the service, if this is not + provided "name" will be used. + :param service_tags: Any tags associated with the service. + :param check: The name of the health check to register + :param check_status: The initial status of the check, + must be one of unknown, passing, warning, or critical. + :param check_service: The service that the check is performed against. + :param check_id: Unique identifier for the service. + :param check_notes: An opaque field that is meant to hold human-readable text. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_register node='node1' address='192.168.1.1' + service='redis' service_address='127.0.0.1' service_port='8080' + service_id='redis_server1' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'datacenter' in kwargs: + data['Datacenter'] = kwargs['datacenter'] + + if 'node' in kwargs: + data['Node'] = kwargs['node'] + else: + ret['message'] = 'Required argument node argument is missing.' + ret['res'] = False + return ret + + if 'address' in kwargs: + data['Address'] = kwargs['address'] + else: + ret['message'] = 'Required argument address argument is missing.' + ret['res'] = False + return ret + + if 'service' in kwargs: + data['Service'] = {} + data['Service']['Service'] = kwargs['service'] + + if 'service_address' in kwargs: + data['Service']['Address'] = kwargs['service_address'] + + if 'service_port' in kwargs: + data['Service']['Port'] = kwargs['service_port'] + + if 'service_id' in kwargs: + data['Service']['ID'] = kwargs['service_id'] + + if 'service_tags' in kwargs: + _tags = kwargs['service_tags'] + if not isinstance(_tags, list): + _tags = [_tags] + data['Service']['Tags'] = _tags + + if 'check' in kwargs: + data['Check'] = {} + data['Check']['Name'] = kwargs['check'] + + if 'check_status' in kwargs: + if kwargs['check_status'] not in ('unknown', 'passing', 'warning', 'critical'): + ret['message'] = 'Check status must be unknown, passing, warning, or critical.' + ret['res'] = False + return ret + data['Check']['Status'] = kwargs['check_status'] + + if 'check_service' in kwargs: + data['Check']['ServiceID'] = kwargs['check_service'] + + if 'check_id' in kwargs: + data['Check']['CheckID'] = kwargs['check_id'] + + if 'check_notes' in kwargs: + data['Check']['Notes'] = kwargs['check_notes'] + + function = 'catalog/register' + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + if res['res']: + ret['res'] = True + ret['message'] = ('Catalog registration ' + 'for {0} successful.'.format(kwargs['name'])) + else: + ret['res'] = False + ret['message'] = ('Catalog registration ' + 'for {0} failed.'.format(kwargs['name'])) + return ret + + +def catalog_deregister(consul_url, **kwargs): + ''' + Deregisters a node, service, or check + + :param consul_url: The Consul server URL. + :param node: The node to deregister. + :param datacenter: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param checkid: The ID of the health check to deregister. + :param serviceid: The ID of the service to deregister. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_register node='node1' + serviceid='redis_server1' checkid='redis_check1' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'datacenter' in kwargs: + data['Datacenter'] = kwargs['datacenter'] + + if 'node' in kwargs: + data['Node'] = kwargs['node'] + else: + ret['message'] = 'Node argument required.' + ret['res'] = False + return ret + + if 'checkid' in kwargs: + data['CheckID'] = kwargs['checkid'] + + if 'serviceid' in kwargs: + data['ServiceID'] = kwargs['serviceid'] + + function = 'catalog/deregister' + res = _query(consul_url=consul_url, + function=function, + method='PUT', + data=data) + if res['res']: + ret['res'] = True + ret['message'] = 'Catalog item {0} removed.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = ('Removing Catalog ' + 'item {0} failed.'.format(kwargs['name'])) + return ret + + +def catalog_datacenters(consul_url): + ''' + Return list of available datacenters from catalog. + + :param consul_url: The Consul server URL. + :return: The list of available datacenters. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_datacenters + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'catalog/datacenters' + ret = _query(consul_url=consul_url, + function=function) + return ret + + +def catalog_nodes(consul_url, **kwargs): + ''' + Return list of available nodes from catalog. + + :param consul_url: The Consul server URL. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: The list of available nodes. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_nodes + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'catalog/nodes' + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def catalog_services(consul_url, **kwargs): + ''' + Return list of available services rom catalog. + + :param consul_url: The Consul server URL. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: The list of available services. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_services + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'catalog/services' + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def catalog_service(consul_url, service, **kwargs): + ''' + Information about the registered service. + + :param consul_url: The Consul server URL. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param tag: Filter returned services with tag parameter. + :return: Information about the requested service. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_service service='redis' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + if 'tag' in kwargs: + query_params['tag'] = kwargs['tag'] + + function = 'catalog/service/{0}'.format(service) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def catalog_node(consul_url, node, **kwargs): + ''' + Information about the registered node. + + :param consul_url: The Consul server URL. + :param node: The node to request information about. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: Information about the requested node. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.catalog_service service='redis' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'catalog/node/{0}'.format(node) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def health_node(consul_url, node, **kwargs): + ''' + Health information about the registered node. + + :param consul_url: The Consul server URL. + :param node: The node to request health information about. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: Health information about the requested node. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.health_node node='node1' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'health/node/{0}'.format(node) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def health_checks(consul_url, service, **kwargs): + ''' + Health information about the registered service. + + :param consul_url: The Consul server URL. + :param service: The service to request health information about. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: Health information about the requested node. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.health_checks service='redis1' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + function = 'health/checks/{0}'.format(service) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def health_service(consul_url, service, **kwargs): + ''' + Health information about the registered service. + + :param consul_url: The Consul server URL. + :param service: The service to request health information about. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param tag: Filter returned services with tag parameter. + :param passing: Filter results to only nodes with all + checks in the passing state. + :return: Health information about the requested node. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.health_service service='redis1' + + salt '*' consul.health_service service='redis1' passing='True' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + if 'tag' in kwargs: + query_params['tag'] = kwargs['tag'] + + if 'passing' in kwargs: + query_params['passing'] = kwargs['passing'] + + function = 'health/service/{0}'.format(service) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def health_state(consul_url, state, **kwargs): + ''' + Returns the checks in the state provided on the path. + + :param consul_url: The Consul server URL. + :param state: The state to show checks for. The supported states + are any, unknown, passing, warning, or critical. + The any state is a wildcard that can be used to + return all checks. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :return: The checks in the provided state. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.health_service service='redis1' + + salt '*' consul.health_service service='redis1' passing='True' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params['dc'] = kwargs['dc'] + + if state not in ('any', 'unknown', 'passing', 'warning', 'critical'): + ret['message'] = 'State must be any, unknown, passing, warning, or critical.' + ret['res'] = False + return ret + + function = 'health/state/{0}'.format(state) + ret = _query(consul_url=consul_url, + function=function, + query_params=query_params) + return ret + + +def status_leader(consul_url): + ''' + Returns the current Raft leader + + :param consul_url: The Consul server URL. + :return: The address of the Raft leader. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.status_leader + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'status/leader' + ret = _query(consul_url=consul_url, + function=function) + return ret + + +def status_peers(consul_url): + ''' + Returns the current Raft peer set + + :param consul_url: The Consul server URL. + :return: Retrieves the Raft peers for the + datacenter in which the the agent is running. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.status_peers + + ''' + ret = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + function = 'status/peers' + ret = _query(consul_url=consul_url, + function=function) + return ret + + +def acl_create(consul_url, **kwargs): + ''' + Create a new ACL token. + + :param consul_url: The Consul server URL. + :param name: Meaningful indicator of the ACL's purpose. + :param type: Type is either client or management. A management + token is comparable to a root user and has the + ability to perform any action including creating, + modifying, and deleting ACLs. + :param rules: The Consul server URL. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_create + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'name' in kwargs: + data['Name'] = kwargs['name'] + + if 'type' in kwargs: + data['Type'] = kwargs['type'] + + if 'rules' in kwargs: + data['Rules'] = kwargs['rules'] + + function = 'acl/create' + res = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + + if res['res']: + ret['res'] = True + ret['message'] = 'ACL {0} created.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = ('Removing Catalog ' + 'item {0} failed.'.format(kwargs['name'])) + return ret + + +def acl_update(consul_url, **kwargs): + ''' + Update an ACL token. + + :param consul_url: The Consul server URL. + :param name: Meaningful indicator of the ACL's purpose. + :param id: Unique identifier for the ACL to update. + :param type: Type is either client or management. A management + token is comparable to a root user and has the + ability to perform any action including creating, + modifying, and deleting ACLs. + :param rules: The Consul server URL. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_update + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'id' in kwargs: + data['ID'] = kwargs['id'] + else: + ret['message'] = 'Required paramter "id" is missing.' + ret['res'] = False + return ret + + if 'name' in kwargs: + data['Name'] = kwargs['name'] + + if 'type' in kwargs: + data['Type'] = kwargs['type'] + + if 'rules' in kwargs: + data['Rules'] = kwargs['rules'] + + function = 'acl/update' + res = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + + if res['res']: + ret['res'] = True + ret['message'] = 'ACL {0} created.'.format(kwargs['name']) + else: + ret['res'] = False + ret['message'] = ('Adding ACL ' + '{0} failed.'.format(kwargs['name'])) + + return ret + + +def acl_delete(consul_url, **kwargs): + ''' + Delete an ACL token. + + :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to update. + :return: Boolean & message of success or failure. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_delete id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'id' not in kwargs: + ret['message'] = 'Required paramter "id" is missing.' + ret['res'] = False + return ret + + function = 'acl/delete/{0}'.format(kwargs['id']) + res = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + + if res['res']: + ret['res'] = True + ret['message'] = 'ACL {0} deleted.'.format(kwargs['id']) + else: + ret['res'] = False + ret['message'] = ('Removing ACL ' + '{0} failed.'.format(kwargs['id'])) + + return ret + + +def acl_info(consul_url, **kwargs): + ''' + Information about an ACL token. + + :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to update. + :return: Information about the ACL requested. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_info id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'id' not in kwargs: + ret['message'] = 'Required paramter "id" is missing.' + ret['res'] = False + return ret + + function = 'acl/info/{0}'.format(kwargs['id']) + ret = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + return ret + + +def acl_clone(consul_url, **kwargs): + ''' + Information about an ACL token. + + :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to update. + :return: Boolean, message of success or + failure, and new ID of cloned ACL. + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_info id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'id' not in kwargs: + ret['message'] = 'Required paramter "id" is missing.' + ret['res'] = False + return ret + + function = 'acl/clone/{0}'.format(kwargs['id']) + res = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + if res['res']: + ret['res'] = True + ret['message'] = 'ACL {0} cloned.'.format(kwargs['name']) + ret['ID'] = ret['data'] + else: + ret['res'] = False + ret['message'] = ('Cloning ACL' + 'item {0} failed.'.format(kwargs['name'])) + return ret + + +def acl_list(consul_url, **kwargs): + ''' + List the ACL tokens. + + :param consul_url: The Consul server URL. + :return: List of ACLs + + CLI Example: + + .. code-block:: bash + + salt '*' consul.acl_list + + ''' + ret = {} + data = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'id' not in kwargs: + ret['message'] = 'Required paramter "id" is missing.' + ret['res'] = False + return ret + + function = 'acl/list/'.format(kwargs['id']) + ret = _query(consul_url=consul_url, + data=data, + method='PUT', + function=function) + return ret + + +def event_fire(consul_url, name, **kwargs): + ''' + List the ACL tokens. + + :param consul_url: The Consul server URL. + :param name: The name of the event to fire. + :param dc: By default, the datacenter of the agent is queried; + however, the dc can be provided using the "dc" parameter. + :param node: Filter by node name. + :param service: Filter by service name. + :param tag: Filter by tag name. + :return: List of ACLs + + CLI Example: + + .. code-block:: bash + + salt '*' consul.event_fire name='deploy' + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if not 'name': + ret['message'] = 'Required paramter "name" is missing.' + ret['res'] = False + return ret + + if 'dc' in kwargs: + query_params = kwargs['dc'] + + if 'node' in kwargs: + query_params = kwargs['node'] + + if 'service' in kwargs: + query_params = kwargs['service'] + + if 'tag' in kwargs: + query_params = kwargs['tag'] + + function = 'event/fire/'.format(name) + res = _query(consul_url=consul_url, + query_params=query_params, + method='PUT', + function=function) + + if res['res']: + ret['res'] = True + ret['message'] = 'Event {0} fired.'.format(name) + ret['data'] = ret['data'] + else: + ret['res'] = False + ret['message'] = ('Cloning ACL' + 'item {0} failed.'.format(kwargs['name'])) + return ret + + +def event_list(consul_url, **kwargs): + ''' + List the recent events. + + :param consul_url: The Consul server URL. + :param name: The name of the event to fire. + :return: List of ACLs + + CLI Example: + + .. code-block:: bash + + salt '*' consul.event_list + + ''' + ret = {} + query_params = {} + if not consul_url: + try: + options = __salt__['config.option']('consul') + if not consul_url: + consul_url = options.get('url') + except (NameError, KeyError, AttributeError): + log.error('No Consul URL found.') + ret['message'] = 'No Consul URL found.' + ret['res'] = False + return ret + + if 'name' in kwargs: + query_params = kwargs['name'] + + function = 'event/list/' + ret = _query(consul_url=consul_url, + query_params=query_params, + function=function) + return ret From 4e2a2f6377c176f71cadda1908ee594622ab3a43 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 7 May 2015 21:48:04 -0700 Subject: [PATCH 352/769] Lint fixes. --- src/saltext/consul/modules/consul.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index db72c10..28826ff 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -280,8 +280,7 @@ def put(consul_url, key, value, **kwargs): if 'acquire' in kwargs: if kwargs['acquire'] not in available_sessions: - ret['message'] = ('{0} is not a valid ', - 'session.'.format(kwargs['acquire'])) + ret['message'] = '{0} is not a valid session.'.format(kwargs['acquire']) ret['res'] = False return ret @@ -293,14 +292,12 @@ def put(consul_url, key, value, **kwargs): if _current['data']['Session'] == kwargs['release']: query_params['release'] = kwargs['release'] else: - ret['message'] = ('{0} locked by ', - 'another session.'.format(key)) + ret['message'] = '{0} locked by another session.'.format(key) ret['res'] = False return ret else: - ret['message'] = ('{0} is not a valid ', - 'session.'.format(kwargs['acquire'])) + ret['message'] = '{0} is not a valid session.'.format(kwargs['acquire']) ret['res'] = False else: log.error('Key {0} does not exist. Skipping release.') @@ -2295,7 +2292,7 @@ def acl_list(consul_url, **kwargs): ret['res'] = False return ret - function = 'acl/list/'.format(kwargs['id']) + function = 'acl/list' ret = _query(consul_url=consul_url, data=data, method='PUT', @@ -2353,7 +2350,7 @@ def event_fire(consul_url, name, **kwargs): if 'tag' in kwargs: query_params = kwargs['tag'] - function = 'event/fire/'.format(name) + function = 'event/fire/{0}'.format(name) res = _query(consul_url=consul_url, query_params=query_params, method='PUT', From 946e89c69126ae8ba8cc1dfae44d307f20cc15ff Mon Sep 17 00:00:00 2001 From: vr-jack Date: Mon, 11 May 2015 13:02:04 -0500 Subject: [PATCH 353/769] Update __init__.py pass _pillar_rend=True to compile_template when processing top.sls as well --- src/saltext/consul/pillar/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a07b9c9..0edae78 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -283,7 +283,8 @@ def get_tops(self): ), self.rend, self.opts['renderer'], - self.opts['environment'] + self.opts['environment'], + _pillar_rend=True ) ] else: @@ -298,7 +299,8 @@ def get_tops(self): top, self.rend, self.opts['renderer'], - saltenv=saltenv + saltenv=saltenv, + _pillar_rend=True ) ) except Exception as exc: @@ -335,7 +337,8 @@ def get_tops(self): ).get('dest', False), self.rend, self.opts['renderer'], - saltenv=saltenv + saltenv=saltenv, + _pillar_rend=True ) ) except Exception as exc: From 865cd6d3a71cc89621ca48d1371cc101cbe54fd0 Mon Sep 17 00:00:00 2001 From: vr-jack Date: Mon, 11 May 2015 13:02:04 -0500 Subject: [PATCH 354/769] Update __init__.py pass _pillar_rend=True to compile_template when processing top.sls as well --- src/saltext/consul/pillar/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c3ba000..4ade96b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -214,7 +214,8 @@ def get_tops(self): ), self.rend, self.opts['renderer'], - self.opts['environment'] + self.opts['environment'], + _pillar_rend=True ) ] else: @@ -227,7 +228,8 @@ def get_tops(self): ), self.rend, self.opts['renderer'], - saltenv=saltenv + saltenv=saltenv, + _pillar_rend=True ) ) except Exception as exc: @@ -262,7 +264,8 @@ def get_tops(self): ).get('dest', False), self.rend, self.opts['renderer'], - saltenv=saltenv + saltenv=saltenv, + _pillar_rend=True ) ) except Exception as exc: From 07888594c63eb06774b895963823263d201ec147 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 14 May 2015 18:32:47 +0300 Subject: [PATCH 355/769] Support pillarenv cmdline in state.sls --- src/saltext/consul/pillar/__init__.py | 36 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0edae78..a95dc44 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -32,7 +32,7 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, - pillar=None): + pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -49,12 +49,12 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, 'local': Pillar }.get(opts['file_client'], Pillar) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, - pillar=pillar) + pillar=pillar, pillarenv=pillarenv) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, - pillar=None): + pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -71,7 +71,7 @@ def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs= #'local': AsyncPillar # TODO: implement }.get(opts['file_client'], Pillar) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, - pillar=pillar) + pillar=pillar, pillarenv=pillarenv) class AsyncRemotePillar(object): @@ -79,13 +79,14 @@ class AsyncRemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, - pillar=None): + pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains self.id_ = id_ self.channel = salt.transport.client.AsyncReqChannel.factory(opts) + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -101,6 +102,7 @@ def compile_pillar(self): load = {'id': self.id_, 'grains': self.grains, 'saltenv': self.opts['environment'], + 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, 'ver': '2', 'cmd': '_pillar'} @@ -125,13 +127,14 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, - pillar=None): + pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains self.id_ = id_ self.channel = salt.transport.Channel.factory(opts) + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -146,6 +149,7 @@ def compile_pillar(self): load = {'id': self.id_, 'grains': self.grains, 'saltenv': self.opts['environment'], + 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, 'ver': '2', 'cmd': '_pillar'} @@ -169,11 +173,11 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, - pillar=None): + pillar=None, pillarenv=None): # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) + self.opts = self.__gen_opts(opts, grains, id_, saltenv=saltenv, ext=ext, pillarenv=pillarenv) self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -219,7 +223,7 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): + def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' @@ -242,6 +246,8 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): opts['id'] = id_ if 'environment' not in opts: opts['environment'] = saltenv + if pillarenv not in opts: + opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): @@ -274,16 +280,16 @@ def get_tops(self): errors = [] # Gather initial top files try: - if self.opts['environment']: - tops[self.opts['environment']] = [ + if self.opts['pillarenv']: + tops[self.opts['pillarenv']] = [ compile_template( self.client.cache_file( self.opts['state_top'], - self.opts['environment'] + self.opts['pillarenv'] ), self.rend, self.opts['renderer'], - self.opts['environment'], + self.opts['pillarenv'] _pillar_rend=True ) ] @@ -426,8 +432,8 @@ def top_matches(self, top): ''' matches = {} for saltenv, body in six.iteritems(top): - if self.opts['environment']: - if saltenv != self.opts['environment']: + if self.opts['pillarenv']: + if saltenv != self.opts['pillarenv']: continue for match, data in six.iteritems(body): if self.matcher.confirm_top( From ff4b84a31fe9c5388031ee98d32c6d66f395e34a Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 15 May 2015 17:07:49 +0300 Subject: [PATCH 356/769] Bugfixes and state module update --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a95dc44..67eaf3d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -289,7 +289,7 @@ def get_tops(self): ), self.rend, self.opts['renderer'], - self.opts['pillarenv'] + self.opts['pillarenv'], _pillar_rend=True ) ] From bb2b564c0a94dabfbc4dbfc3fa76e1cf6639d078 Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 19:06:53 +0100 Subject: [PATCH 357/769] Use consul as an external pillar source. Requires python-consul --- src/saltext/consul/pillar/consul_pillar.py | 199 +++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 src/saltext/consul/pillar/consul_pillar.py diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py new file mode 100644 index 0000000..dc00e17 --- /dev/null +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +''' +Use consul data as a Pillar source + +:depends: - python-consul + +In order to use an consul server, a profile must be created in the master +configuration file: + +.. code-block:: yaml + + my_consul_config: + consul.host: 127.0.0.1 + consul.port: 8500 + +After the profile is created, configure the external pillar system to use it. +Optionally, a root may be specified. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config + + ext_pillar: + - consul: my_consul_config root=/salt + +Using these configuration profiles, multiple consul sources may also be used: + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config + - consul: my_other_consul_config + +The ``minion_id`` may be used in the ``root`` path to expose minion-specific +information stored in consul. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config root=/salt/%(minion_id)s + +Minion-specific values may override shared values when the minion-specific root +appears after the shared root: + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config root=/salt-shared + - consul: my_other_consul_config root=/salt-private/%(minion_id)s + +''' +from __future__ import absolute_import + +# Import python libs +import logging + +import re + +from salt.exceptions import CommandExecutionError + +# Import third party libs +try: + import consul + HAS_CONSUL = True +except ImportError: + HAS_CONSUL = False + +__virtualname__ = 'consul' + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only return if python-consul is installed + ''' + return __virtualname__ if HAS_CONSUL else False + + +def ext_pillar(minion_id, + pillar, # pylint: disable=W0613 + conf): + ''' + Check consul for all data + ''' + comps = conf.split() + + profile = None + if comps[0]: + profile = comps[0] + client = get_conn(__opts__, profile) + + path = '' + if len(comps) > 1 and comps[1].startswith('root='): + path = comps[1].replace('root=', '') + + # put the minion's ID in the path if necessary + path %= { + 'minion_id': minion_id + } + + try: + pillar = fetch_tree(client, path) + except KeyError: + log.error('No such key in consul profile {0}: {1}'.format(profile, path)) + pillar = {} + + return pillar + +def consul_fetch(client, path): + ''' + Query consul for all keys/values within base path + ''' + return client.kv.get(path, recurse=True) + +def fetch_tree(client, path): + ''' + Grab data from consul, trim base path and remove any keys which + are folders. Take the remaining data and send it to be formatted + in such a way as to be used as pillar data. + ''' + index, items = consul_fetch(client, path) + ret = {} + has_children = re.compile(r'/$') + + log.debug('Fetched items: %r', format(items)) + + for item in reversed(items): + key = re.sub(r'^' + path + '/?', '', item['Key']) + if key != "": + log.debug('key/path - {0}: {1}'.format(path, key)) + log.debug('has_children? %r', format(has_children.search(key))) + if has_children.search(key) == None: + ret = pillar_format(ret, key.split('/'), item['Value']) + log.debug('Fetching subkeys for key: %r', format(item)) + + return ret + +def pillar_format(ret, keys, value): + ''' + Perform data formatting to be used as pillar data and + merge it with the current pillar data + ''' + Get t + if value == None: + return ret + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data + keyvalue = keys.pop() + pil = {keyvalue: pillar_value} + keys.reverse() + for k in keys: + pil = {k: pil} + + return dict_merge(ret, pil) + +def dict_merge(d1, d2): + ''' + Take 2 dictionaries and deep merge them + ''' + master = d1.copy() + for (k, v) in d2.iteritems(): + if k in master and isinstance(master[k], dict): + master[k] = dict_merge(master[k], v) + else: + master[k] = v + return master + +def get_conn(opts, profile): + + ''' + Return a client object for accessing consul + ''' + opts_pillar = opts.get('pillar', {}) + opts_master = opts_pillar.get('master', {}) + + opts_merged = {} + opts_merged.update(opts_master) + opts_merged.update(opts_pillar) + opts_merged.update(opts) + + if profile: + conf = opts_merged.get(profile, {}) + else: + conf = opts_merged + + consul_host = conf.get('consul.host', '127.0.0.1') + consul_port = conf.get('consul.port', 8500) + + if HAS_CONSUL: + return consul.Consul(host=consul_host, port=consul_port) + else: + raise CommandExecutionError( + '(unable to import consul, ' + 'module most likely not installed. Download python-consul ' + 'module and be sure to import consul)' + ) From 126541757f3e429d44d332d13dd076b6ba5cc63f Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 20:19:11 +0100 Subject: [PATCH 358/769] Fixed a typo --- src/saltext/consul/pillar/consul_pillar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index dc00e17..5635e0c 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -143,7 +143,6 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' - Get t if value == None: return ret array_data = value.split('\n') From 2326b8c3b8f7736191740f0e6181fc19974a2124 Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 22:29:08 +0100 Subject: [PATCH 359/769] Style changes to bring code in line with coding standards --- src/saltext/consul/pillar/consul_pillar.py | 208 +++++++++++---------- 1 file changed, 107 insertions(+), 101 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 5635e0c..6698224 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -61,10 +61,10 @@ # Import third party libs try: - import consul - HAS_CONSUL = True + import consul + HAS_CONSUL = True except ImportError: - HAS_CONSUL = False + HAS_CONSUL = False __virtualname__ = 'consul' @@ -73,126 +73,132 @@ def __virtual__(): - ''' - Only return if python-consul is installed - ''' - return __virtualname__ if HAS_CONSUL else False + ''' + Only return if python-consul is installed + ''' + return __virtualname__ if HAS_CONSUL else False def ext_pillar(minion_id, pillar, # pylint: disable=W0613 conf): - ''' - Check consul for all data - ''' - comps = conf.split() + ''' + Check consul for all data + ''' + comps = conf.split() - profile = None - if comps[0]: - profile = comps[0] - client = get_conn(__opts__, profile) + profile = None + if comps[0]: + profile = comps[0] + client = get_conn(__opts__, profile) - path = '' - if len(comps) > 1 and comps[1].startswith('root='): - path = comps[1].replace('root=', '') + path = '' + if len(comps) > 1 and comps[1].startswith('root='): + path = comps[1].replace('root=', '') - # put the minion's ID in the path if necessary - path %= { - 'minion_id': minion_id - } + # put the minion's ID in the path if necessary + path %= { + 'minion_id': minion_id + } - try: - pillar = fetch_tree(client, path) - except KeyError: - log.error('No such key in consul profile {0}: {1}'.format(profile, path)) - pillar = {} + try: + pillar = fetch_tree(client, path) + except KeyError: + log.error('No such key in consul profile {0}: {1}' + .format(profile, path)) + pillar = {} + + return pillar - return pillar def consul_fetch(client, path): - ''' - Query consul for all keys/values within base path - ''' - return client.kv.get(path, recurse=True) + ''' + Query consul for all keys/values within base path + ''' + return client.kv.get(path, recurse=True) + def fetch_tree(client, path): - ''' - Grab data from consul, trim base path and remove any keys which - are folders. Take the remaining data and send it to be formatted - in such a way as to be used as pillar data. - ''' - index, items = consul_fetch(client, path) - ret = {} - has_children = re.compile(r'/$') - - log.debug('Fetched items: %r', format(items)) - - for item in reversed(items): - key = re.sub(r'^' + path + '/?', '', item['Key']) - if key != "": - log.debug('key/path - {0}: {1}'.format(path, key)) - log.debug('has_children? %r', format(has_children.search(key))) - if has_children.search(key) == None: - ret = pillar_format(ret, key.split('/'), item['Value']) - log.debug('Fetching subkeys for key: %r', format(item)) - - return ret + ''' + Grab data from consul, trim base path and remove any keys which + are folders. Take the remaining data and send it to be formatted + in such a way as to be used as pillar data. + ''' + index, items = consul_fetch(client, path) + ret = {} + has_children = re.compile(r'/$') + + log.debug('Fetched items: %r', format(items)) + + for item in reversed(items): + key = re.sub(r'^' + path + '/?', '', item['Key']) + if key != "": + log.debug('key/path - {0}: {1}'.format(path, key)) + log.debug('has_children? %r', format(has_children.search(key))) + if has_children.search(key) is None: + ret = pillar_format(ret, key.split('/'), item['Value']) + log.debug('Fetching subkeys for key: %r', format(item)) -def pillar_format(ret, keys, value): - ''' - Perform data formatting to be used as pillar data and - merge it with the current pillar data - ''' - if value == None: return ret - array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data - keyvalue = keys.pop() - pil = {keyvalue: pillar_value} - keys.reverse() - for k in keys: - pil = {k: pil} - return dict_merge(ret, pil) + +def pillar_format(ret, keys, value): + ''' + Perform data formatting to be used as pillar data and + merge it with the current pillar data + ''' + if value is None: + return ret + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data + keyvalue = keys.pop() + pil = {keyvalue: pillar_value} + keys.reverse() + for k in keys: + pil = {k: pil} + + return dict_merge(ret, pil) + def dict_merge(d1, d2): - ''' - Take 2 dictionaries and deep merge them - ''' - master = d1.copy() - for (k, v) in d2.iteritems(): - if k in master and isinstance(master[k], dict): - master[k] = dict_merge(master[k], v) - else: - master[k] = v - return master + ''' + Take 2 dictionaries and deep merge them + ''' + master = d1.copy() + for (k, v) in d2.iteritems(): + if k in master and isinstance(master[k], dict): + master[k] = dict_merge(master[k], v) + else: + master[k] = v + return master + def get_conn(opts, profile): - ''' - Return a client object for accessing consul - ''' - opts_pillar = opts.get('pillar', {}) - opts_master = opts_pillar.get('master', {}) - - opts_merged = {} - opts_merged.update(opts_master) - opts_merged.update(opts_pillar) - opts_merged.update(opts) - - if profile: - conf = opts_merged.get(profile, {}) - else: - conf = opts_merged - - consul_host = conf.get('consul.host', '127.0.0.1') - consul_port = conf.get('consul.port', 8500) - - if HAS_CONSUL: - return consul.Consul(host=consul_host, port=consul_port) - else: - raise CommandExecutionError( + ''' + Return a client object for accessing consul + ''' + opts_pillar = opts.get('pillar', {}) + opts_master = opts_pillar.get('master', {}) + + opts_merged = {} + opts_merged.update(opts_master) + opts_merged.update(opts_pillar) + opts_merged.update(opts) + + if profile: + conf = opts_merged.get(profile, {}) + else: + conf = opts_merged + + consul_host = conf.get('consul.host', '127.0.0.1') + consul_port = conf.get('consul.port', 8500) + + if HAS_CONSUL: + return consul.Consul(host=consul_host, port=consul_port) + else: + raise CommandExecutionError( '(unable to import consul, ' 'module most likely not installed. Download python-consul ' 'module and be sure to import consul)' - ) + ) From cd32af087d286fecae60bd7fc23f12db892b8eb6 Mon Sep 17 00:00:00 2001 From: Arnold Bechtoldt Date: Thu, 4 Jun 2015 21:03:02 +0200 Subject: [PATCH 360/769] add missing module documentation to references --- docs/ref/modules/all/salt.modules.consul.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/ref/modules/all/salt.modules.consul.rst diff --git a/docs/ref/modules/all/salt.modules.consul.rst b/docs/ref/modules/all/salt.modules.consul.rst new file mode 100644 index 0000000..4567678 --- /dev/null +++ b/docs/ref/modules/all/salt.modules.consul.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.consul +=================== + +.. automodule:: salt.modules.consul + :members: From 056984f797dd4ac11d0a5c11253b8d24b6a01889 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 5 Jun 2015 21:07:45 -0700 Subject: [PATCH 361/769] Updating consul module to use salt.utils.http instead of requests. --- src/saltext/consul/modules/consul.py | 216 ++++++++++++++++----------- 1 file changed, 129 insertions(+), 87 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 28826ff..3d16e18 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -16,24 +16,20 @@ # pylint: enable=import-error,no-name-in-module import base64 - -try: - import requests - from requests.exceptions import ConnectionError - ENABLED = True -except ImportError: - ENABLED = False +import json import logging log = logging.getLogger(__name__) +from salt.exceptions import SaltInvocationError + __virtualname__ = 'consul' def _query(function, consul_url, - api_version='v1', method='GET', + api_version='v1', data=None, query_params=None): ''' @@ -50,39 +46,29 @@ def _query(function, if not query_params: query_params = {} - if data is None: - data = {} - ret = {'data': '', 'res': True} base_url = _urljoin(consul_url, '{0}/'.format(api_version)) url = _urljoin(base_url, function, False) - try: - result = requests.request( - method=method, - url=url, - headers=headers, - params=query_params, - data=data, - verify=True, - ) - except ConnectionError as e: - ret['data'] = e - ret['res'] = False - return ret - - if result.status_code == salt.ext.six.moves.http_client.OK: - result = result.json() - if result: - ret['data'] = result - ret['res'] = True - else: - ret['res'] = False - elif result.status_code == salt.ext.six.moves.http_client.NO_CONTENT: + result = salt.utils.http.query( + url, + method=method, + params=query_params, + data=data, + decode=True, + status=True, + header_dict=headers, + opts=__opts__, + ) + + if result.get('status', None) == salt.ext.six.moves.http_client.OK: + ret['data'] = result['dict'] + ret['res'] = True + elif result.get('status', None) == salt.ext.six.moves.http_client.NO_CONTENT: ret['res'] = False - elif result.status_code == salt.ext.six.moves.http_client.NOT_FOUND: + elif result.get('status', None) == salt.ext.six.moves.http_client.NOT_FOUND: ret['data'] = 'Key not found.' ret['res'] = False else: @@ -95,7 +81,7 @@ def _query(function, return ret -def list(consul_url, key=None, **kwargs): +def list(consul_url=None, key=None, **kwargs): ''' List keys in Consul @@ -144,7 +130,7 @@ def list(consul_url, key=None, **kwargs): return ret -def get(consul_url, key, recurse=False, decode=False, raw=False): +def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): ''' Get key from Consul @@ -187,6 +173,9 @@ def get(consul_url, key, recurse=False, decode=False, raw=False): ret['res'] = False return ret + if not key: + raise SaltInvocationError('Required argument "key" is missing.') + query_params = {} function = 'kv/{0}'.format(key) if recurse: @@ -204,7 +193,7 @@ def get(consul_url, key, recurse=False, decode=False, raw=False): return ret -def put(consul_url, key, value, **kwargs): +def put(consul_url=None, key=None, value=None, **kwargs): ''' Put values into Consul @@ -247,6 +236,9 @@ def put(consul_url, key, value, **kwargs): ret['res'] = False return ret + if not key: + raise SaltInvocationError('Required argument "key" is missing.') + query_params = {} available_sessions = session_list(consul_url=consul_url, return_list=True) @@ -304,11 +296,13 @@ def put(consul_url, key, value, **kwargs): data = value function = 'kv/{0}'.format(key) + method = 'PUT' ret = _query(consul_url=consul_url, function=function, - method='PUT', - data=data, + method=method, + data=json.dumps(data), query_params=query_params) + if ret['res']: ret['res'] = True ret['data'] = 'Added key {0} with value {1}.'.format(key, value) @@ -318,7 +312,7 @@ def put(consul_url, key, value, **kwargs): return ret -def delete(consul_url, key, **kwargs): +def delete(consul_url=None, key=None, **kwargs): ''' Delete values from Consul @@ -350,6 +344,9 @@ def delete(consul_url, key, **kwargs): ret['res'] = False return ret + if not key: + raise SaltInvocationError('Required argument "key" is missing.') + query_params = {} if 'recurse' in kwargs: @@ -379,7 +376,7 @@ def delete(consul_url, key, **kwargs): return ret -def agent_checks(consul_url): +def agent_checks(consul_url=None): ''' Returns the checks the local agent is managing @@ -412,7 +409,7 @@ def agent_checks(consul_url): return ret -def agent_services(consul_url): +def agent_services(consul_url=None): ''' Returns the services the local agent is managing @@ -445,7 +442,7 @@ def agent_services(consul_url): return ret -def agent_members(consul_url, **kwargs): +def agent_members(consul_url=None, **kwargs): ''' Returns the members as seen by the local serf agent @@ -483,7 +480,7 @@ def agent_members(consul_url, **kwargs): return ret -def agent_self(consul_url): +def agent_self(consul_url=None): ''' Returns the local node configuration @@ -518,7 +515,7 @@ def agent_self(consul_url): return ret -def agent_maintenance(consul_url, **kwargs): +def agent_maintenance(consul_url=None, **kwargs): ''' Manages node maintenance mode @@ -577,7 +574,7 @@ def agent_maintenance(consul_url, **kwargs): return ret -def agent_join(consul_url, address, **kwargs): +def agent_join(consul_url=None, address=None, **kwargs): ''' Triggers the local agent to join a node @@ -606,6 +603,9 @@ def agent_join(consul_url, address, **kwargs): ret['res'] = False return ret + if not address: + raise SaltInvocationError('Required argument "address" is missing.') + if 'wan' in kwargs: query_params['wan'] = kwargs['wan'] @@ -624,7 +624,7 @@ def agent_join(consul_url, address, **kwargs): return ret -def agent_leave(consul_url, node): +def agent_leave(consul_url=None, node=None): ''' Used to instruct the agent to force a node into the left state. @@ -652,6 +652,9 @@ def agent_leave(consul_url, node): ret['res'] = False return ret + if not node: + raise SaltInvocationError('Required argument "node" is missing.') + function = 'agent/force-leave/{0}'.format(node) res = _query(consul_url=consul_url, function=function, @@ -666,7 +669,7 @@ def agent_leave(consul_url, node): return ret -def agent_check_register(consul_url, **kwargs): +def agent_check_register(consul_url=None, **kwargs): ''' The register endpoint is used to add a new check to the local agent. @@ -755,7 +758,7 @@ def agent_check_register(consul_url, **kwargs): return ret -def agent_check_deregister(consul_url, checkid): +def agent_check_deregister(consul_url=None, checkid=None): ''' The agent will take care of deregistering the check from the Catalog. @@ -767,8 +770,7 @@ def agent_check_deregister(consul_url, checkid): .. code-block:: bash - salt '*' consul.agent_check_register name='Memory Utilization' - script='/usr/local/bin/check_mem.py' interval='15s' + salt '*' consul.agent_check_deregister checkid='Memory Utilization' ''' ret = {} @@ -783,6 +785,9 @@ def agent_check_deregister(consul_url, checkid): ret['res'] = False return ret + if not checkid: + raise SaltInvocationError('Required argument "checkid" is missing.') + function = 'agent/check/deregister/{0}'.format(checkid) res = _query(consul_url=consul_url, function=function, @@ -796,7 +801,7 @@ def agent_check_deregister(consul_url, checkid): return ret -def agent_check_pass(consul_url, checkid, **kwargs): +def agent_check_pass(consul_url=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to passing and the TTL @@ -828,6 +833,9 @@ def agent_check_pass(consul_url, checkid, **kwargs): ret['res'] = False return ret + if not checkid: + raise SaltInvocationError('Required argument "checkid" is missing.') + if 'note' in kwargs: query_params['note'] = kwargs['note'] @@ -845,7 +853,7 @@ def agent_check_pass(consul_url, checkid, **kwargs): return ret -def agent_check_warn(consul_url, checkid, **kwargs): +def agent_check_warn(consul_url=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to warning and the TTL @@ -877,6 +885,9 @@ def agent_check_warn(consul_url, checkid, **kwargs): ret['res'] = False return ret + if not checkid: + raise SaltInvocationError('Required argument "checkid" is missing.') + if 'note' in kwargs: query_params['note'] = kwargs['note'] @@ -894,7 +905,7 @@ def agent_check_warn(consul_url, checkid, **kwargs): return ret -def agent_check_fail(consul_url, checkid, **kwargs): +def agent_check_fail(consul_url=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to critical and the @@ -926,6 +937,9 @@ def agent_check_fail(consul_url, checkid, **kwargs): ret['res'] = False return ret + if not checkid: + raise SaltInvocationError('Required argument "checkid" is missing.') + if 'note' in kwargs: query_params['note'] = kwargs['note'] @@ -943,7 +957,7 @@ def agent_check_fail(consul_url, checkid, **kwargs): return ret -def agent_service_register(consul_url, **kwargs): +def agent_service_register(consul_url=None, **kwargs): ''' The used to add a new service, with an optional health check, to the local agent. @@ -1045,7 +1059,7 @@ def agent_service_register(consul_url, **kwargs): return ret -def agent_service_deregister(consul_url, serviceid): +def agent_service_deregister(consul_url=None, serviceid=None): ''' Used to remove a service. @@ -1073,6 +1087,9 @@ def agent_service_deregister(consul_url, serviceid): ret['res'] = False return ret + if not serviceid: + raise SaltInvocationError('Required argument "serviceid" is missing.') + function = 'agent/service/deregister/{0}'.format(serviceid) res = _query(consul_url=consul_url, function=function, @@ -1087,7 +1104,7 @@ def agent_service_deregister(consul_url, serviceid): return ret -def agent_service_maintenance(consul_url, serviceid, **kwargs): +def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): ''' Used to place a service into maintenance mode. @@ -1119,6 +1136,9 @@ def agent_service_maintenance(consul_url, serviceid, **kwargs): ret['res'] = False return ret + if not serviceid: + raise SaltInvocationError('Required argument "serviceid" is missing.') + if 'enable' in kwargs: query_params['enable'] = kwargs['enable'] else: @@ -1145,7 +1165,7 @@ def agent_service_maintenance(consul_url, serviceid, **kwargs): return ret -def session_create(consul_url, **kwargs): +def session_create(consul_url=None, **kwargs): ''' Used to create a session. @@ -1237,7 +1257,7 @@ def session_create(consul_url, **kwargs): return ret -def session_list(consul_url, return_list=False, **kwargs): +def session_list(consul_url=None, return_list=False, **kwargs): ''' Used to list sessions. @@ -1286,7 +1306,7 @@ def session_list(consul_url, return_list=False, **kwargs): return ret -def session_destroy(consul_url, session, **kwargs): +def session_destroy(consul_url=None, session=None, **kwargs): ''' Destroy session @@ -1315,6 +1335,9 @@ def session_destroy(consul_url, session, **kwargs): ret['res'] = False return ret + if not session: + raise SaltInvocationError('Required argument "session" is missing.') + query_params = {} if 'dc' in kwargs: @@ -1333,7 +1356,7 @@ def session_destroy(consul_url, session, **kwargs): return ret -def session_info(consul_url, session, **kwargs): +def session_info(consul_url=None, session=None, **kwargs): ''' Information about a session @@ -1362,6 +1385,9 @@ def session_info(consul_url, session, **kwargs): ret['res'] = False return ret + if not session: + raise SaltInvocationError('Required argument "session" is missing.') + query_params = {} if 'dc' in kwargs: @@ -1374,7 +1400,7 @@ def session_info(consul_url, session, **kwargs): return ret -def catalog_register(consul_url, **kwargs): +def catalog_register(consul_url=None, **kwargs): ''' Registers a new node, service, or check @@ -1491,7 +1517,7 @@ def catalog_register(consul_url, **kwargs): return ret -def catalog_deregister(consul_url, **kwargs): +def catalog_deregister(consul_url=None, **kwargs): ''' Deregisters a node, service, or check @@ -1555,7 +1581,7 @@ def catalog_deregister(consul_url, **kwargs): return ret -def catalog_datacenters(consul_url): +def catalog_datacenters(consul_url=None): ''' Return list of available datacenters from catalog. @@ -1587,7 +1613,7 @@ def catalog_datacenters(consul_url): return ret -def catalog_nodes(consul_url, **kwargs): +def catalog_nodes(consul_url=None, **kwargs): ''' Return list of available nodes from catalog. @@ -1626,7 +1652,7 @@ def catalog_nodes(consul_url, **kwargs): return ret -def catalog_services(consul_url, **kwargs): +def catalog_services(consul_url=None, **kwargs): ''' Return list of available services rom catalog. @@ -1665,7 +1691,7 @@ def catalog_services(consul_url, **kwargs): return ret -def catalog_service(consul_url, service, **kwargs): +def catalog_service(consul_url=None, service=None, **kwargs): ''' Information about the registered service. @@ -1695,6 +1721,9 @@ def catalog_service(consul_url, service, **kwargs): ret['res'] = False return ret + if not service: + raise SaltInvocationError('Required argument "service" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1708,7 +1737,7 @@ def catalog_service(consul_url, service, **kwargs): return ret -def catalog_node(consul_url, node, **kwargs): +def catalog_node(consul_url=None, node=None, **kwargs): ''' Information about the registered node. @@ -1738,6 +1767,9 @@ def catalog_node(consul_url, node, **kwargs): ret['res'] = False return ret + if not node: + raise SaltInvocationError('Required argument "node" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1748,7 +1780,7 @@ def catalog_node(consul_url, node, **kwargs): return ret -def health_node(consul_url, node, **kwargs): +def health_node(consul_url=None, node=None, **kwargs): ''' Health information about the registered node. @@ -1778,6 +1810,9 @@ def health_node(consul_url, node, **kwargs): ret['res'] = False return ret + if not node: + raise SaltInvocationError('Required argument "node" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1788,7 +1823,7 @@ def health_node(consul_url, node, **kwargs): return ret -def health_checks(consul_url, service, **kwargs): +def health_checks(consul_url=None, service=None, **kwargs): ''' Health information about the registered service. @@ -1818,6 +1853,9 @@ def health_checks(consul_url, service, **kwargs): ret['res'] = False return ret + if not node: + raise SaltInvocationError('Required argument "node" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1828,7 +1866,7 @@ def health_checks(consul_url, service, **kwargs): return ret -def health_service(consul_url, service, **kwargs): +def health_service(consul_url=None, service=None, **kwargs): ''' Health information about the registered service. @@ -1863,6 +1901,9 @@ def health_service(consul_url, service, **kwargs): ret['res'] = False return ret + if not service: + raise SaltInvocationError('Required argument "service" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1879,7 +1920,7 @@ def health_service(consul_url, service, **kwargs): return ret -def health_state(consul_url, state, **kwargs): +def health_state(consul_url=None, state=None, **kwargs): ''' Returns the checks in the state provided on the path. @@ -1896,9 +1937,9 @@ def health_state(consul_url, state, **kwargs): .. code-block:: bash - salt '*' consul.health_service service='redis1' + salt '*' consul.health_state state='redis1' - salt '*' consul.health_service service='redis1' passing='True' + salt '*' consul.health_state service='redis1' passing='True' ''' ret = {} @@ -1914,6 +1955,9 @@ def health_state(consul_url, state, **kwargs): ret['res'] = False return ret + if not state: + raise SaltInvocationError('Required argument "state" is missing.') + if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] @@ -1929,7 +1973,7 @@ def health_state(consul_url, state, **kwargs): return ret -def status_leader(consul_url): +def status_leader(consul_url=None): ''' Returns the current Raft leader @@ -1994,7 +2038,7 @@ def status_peers(consul_url): return ret -def acl_create(consul_url, **kwargs): +def acl_create(consul_url=None, **kwargs): ''' Create a new ACL token. @@ -2052,7 +2096,7 @@ def acl_create(consul_url, **kwargs): return ret -def acl_update(consul_url, **kwargs): +def acl_update(consul_url=None, **kwargs): ''' Update an ACL token. @@ -2119,7 +2163,7 @@ def acl_update(consul_url, **kwargs): return ret -def acl_delete(consul_url, **kwargs): +def acl_delete(consul_url=None, **kwargs): ''' Delete an ACL token. @@ -2169,7 +2213,7 @@ def acl_delete(consul_url, **kwargs): return ret -def acl_info(consul_url, **kwargs): +def acl_info(consul_url=None, **kwargs): ''' Information about an ACL token. @@ -2210,7 +2254,7 @@ def acl_info(consul_url, **kwargs): return ret -def acl_clone(consul_url, **kwargs): +def acl_clone(consul_url=None, **kwargs): ''' Information about an ACL token. @@ -2260,7 +2304,7 @@ def acl_clone(consul_url, **kwargs): return ret -def acl_list(consul_url, **kwargs): +def acl_list(consul_url=None, **kwargs): ''' List the ACL tokens. @@ -2300,7 +2344,7 @@ def acl_list(consul_url, **kwargs): return ret -def event_fire(consul_url, name, **kwargs): +def event_fire(consul_url=None, name=None, **kwargs): ''' List the ACL tokens. @@ -2333,10 +2377,8 @@ def event_fire(consul_url, name, **kwargs): ret['res'] = False return ret - if not 'name': - ret['message'] = 'Required paramter "name" is missing.' - ret['res'] = False - return ret + if not name: + raise SaltInvocationError('Required argument "name" is missing.') if 'dc' in kwargs: query_params = kwargs['dc'] @@ -2367,7 +2409,7 @@ def event_fire(consul_url, name, **kwargs): return ret -def event_list(consul_url, **kwargs): +def event_list(consul_url=None, **kwargs): ''' List the recent events. From ced089ac2c43bd32f36503eeb7a4da226d2336da Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 5 Jun 2015 22:33:10 -0700 Subject: [PATCH 362/769] lint. wrong variable check in a check. --- src/saltext/consul/modules/consul.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 3d16e18..13db3bb 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1853,8 +1853,8 @@ def health_checks(consul_url=None, service=None, **kwargs): ret['res'] = False return ret - if not node: - raise SaltInvocationError('Required argument "node" is missing.') + if not service: + raise SaltInvocationError('Required argument "service" is missing.') if 'dc' in kwargs: query_params['dc'] = kwargs['dc'] From 8396a418157019e160b9a1871e030bbdce3d26d2 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 8 Jun 2015 08:16:25 -0700 Subject: [PATCH 363/769] Missing import for salt.utils.http --- src/saltext/consul/modules/consul.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 13db3bb..0cc42be 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -15,6 +15,9 @@ import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module +# Import salt libs +import salt.utils.http + import base64 import json From 2d363bfd199cdef4594290bf3b71caf8cc19f4e6 Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 9 Jun 2015 17:05:38 +0100 Subject: [PATCH 364/769] Don't split multiline values into a list if wrapped inside " Use salt.utils.dictupdate to merge dictionaries --- src/saltext/consul/pillar/consul_pillar.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 6698224..df9b618 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -58,6 +58,7 @@ import re from salt.exceptions import CommandExecutionError +from salt.utils.dictupdate import update as dict_merge # Import third party libs try: @@ -149,8 +150,11 @@ def pillar_format(ret, keys, value): ''' if value is None: return ret - array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data + if value[0] == value[-1] == '"': + pillar_value = value[1:-1] + else: + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data keyvalue = keys.pop() pil = {keyvalue: pillar_value} keys.reverse() @@ -160,19 +164,6 @@ def pillar_format(ret, keys, value): return dict_merge(ret, pil) -def dict_merge(d1, d2): - ''' - Take 2 dictionaries and deep merge them - ''' - master = d1.copy() - for (k, v) in d2.iteritems(): - if k in master and isinstance(master[k], dict): - master[k] = dict_merge(master[k], v) - else: - master[k] = v - return master - - def get_conn(opts, profile): ''' From 4013eef687af550e1053f9e3635035db0766fcdc Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 19:06:53 +0100 Subject: [PATCH 365/769] Use consul as an external pillar source. Requires python-consul --- src/saltext/consul/pillar/consul_pillar.py | 199 +++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 src/saltext/consul/pillar/consul_pillar.py diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py new file mode 100644 index 0000000..dc00e17 --- /dev/null +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +''' +Use consul data as a Pillar source + +:depends: - python-consul + +In order to use an consul server, a profile must be created in the master +configuration file: + +.. code-block:: yaml + + my_consul_config: + consul.host: 127.0.0.1 + consul.port: 8500 + +After the profile is created, configure the external pillar system to use it. +Optionally, a root may be specified. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config + + ext_pillar: + - consul: my_consul_config root=/salt + +Using these configuration profiles, multiple consul sources may also be used: + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config + - consul: my_other_consul_config + +The ``minion_id`` may be used in the ``root`` path to expose minion-specific +information stored in consul. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config root=/salt/%(minion_id)s + +Minion-specific values may override shared values when the minion-specific root +appears after the shared root: + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config root=/salt-shared + - consul: my_other_consul_config root=/salt-private/%(minion_id)s + +''' +from __future__ import absolute_import + +# Import python libs +import logging + +import re + +from salt.exceptions import CommandExecutionError + +# Import third party libs +try: + import consul + HAS_CONSUL = True +except ImportError: + HAS_CONSUL = False + +__virtualname__ = 'consul' + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only return if python-consul is installed + ''' + return __virtualname__ if HAS_CONSUL else False + + +def ext_pillar(minion_id, + pillar, # pylint: disable=W0613 + conf): + ''' + Check consul for all data + ''' + comps = conf.split() + + profile = None + if comps[0]: + profile = comps[0] + client = get_conn(__opts__, profile) + + path = '' + if len(comps) > 1 and comps[1].startswith('root='): + path = comps[1].replace('root=', '') + + # put the minion's ID in the path if necessary + path %= { + 'minion_id': minion_id + } + + try: + pillar = fetch_tree(client, path) + except KeyError: + log.error('No such key in consul profile {0}: {1}'.format(profile, path)) + pillar = {} + + return pillar + +def consul_fetch(client, path): + ''' + Query consul for all keys/values within base path + ''' + return client.kv.get(path, recurse=True) + +def fetch_tree(client, path): + ''' + Grab data from consul, trim base path and remove any keys which + are folders. Take the remaining data and send it to be formatted + in such a way as to be used as pillar data. + ''' + index, items = consul_fetch(client, path) + ret = {} + has_children = re.compile(r'/$') + + log.debug('Fetched items: %r', format(items)) + + for item in reversed(items): + key = re.sub(r'^' + path + '/?', '', item['Key']) + if key != "": + log.debug('key/path - {0}: {1}'.format(path, key)) + log.debug('has_children? %r', format(has_children.search(key))) + if has_children.search(key) == None: + ret = pillar_format(ret, key.split('/'), item['Value']) + log.debug('Fetching subkeys for key: %r', format(item)) + + return ret + +def pillar_format(ret, keys, value): + ''' + Perform data formatting to be used as pillar data and + merge it with the current pillar data + ''' + Get t + if value == None: + return ret + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data + keyvalue = keys.pop() + pil = {keyvalue: pillar_value} + keys.reverse() + for k in keys: + pil = {k: pil} + + return dict_merge(ret, pil) + +def dict_merge(d1, d2): + ''' + Take 2 dictionaries and deep merge them + ''' + master = d1.copy() + for (k, v) in d2.iteritems(): + if k in master and isinstance(master[k], dict): + master[k] = dict_merge(master[k], v) + else: + master[k] = v + return master + +def get_conn(opts, profile): + + ''' + Return a client object for accessing consul + ''' + opts_pillar = opts.get('pillar', {}) + opts_master = opts_pillar.get('master', {}) + + opts_merged = {} + opts_merged.update(opts_master) + opts_merged.update(opts_pillar) + opts_merged.update(opts) + + if profile: + conf = opts_merged.get(profile, {}) + else: + conf = opts_merged + + consul_host = conf.get('consul.host', '127.0.0.1') + consul_port = conf.get('consul.port', 8500) + + if HAS_CONSUL: + return consul.Consul(host=consul_host, port=consul_port) + else: + raise CommandExecutionError( + '(unable to import consul, ' + 'module most likely not installed. Download python-consul ' + 'module and be sure to import consul)' + ) From de597f7c75a5e49bea7c938c82d94092d2f83042 Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 20:19:11 +0100 Subject: [PATCH 366/769] Fixed a typo --- src/saltext/consul/pillar/consul_pillar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index dc00e17..5635e0c 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -143,7 +143,6 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' - Get t if value == None: return ret array_data = value.split('\n') From d6c53a08288714eba8c6b0759c43b27bd9f70cbd Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 2 Jun 2015 22:29:08 +0100 Subject: [PATCH 367/769] Style changes to bring code in line with coding standards --- src/saltext/consul/pillar/consul_pillar.py | 208 +++++++++++---------- 1 file changed, 107 insertions(+), 101 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 5635e0c..6698224 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -61,10 +61,10 @@ # Import third party libs try: - import consul - HAS_CONSUL = True + import consul + HAS_CONSUL = True except ImportError: - HAS_CONSUL = False + HAS_CONSUL = False __virtualname__ = 'consul' @@ -73,126 +73,132 @@ def __virtual__(): - ''' - Only return if python-consul is installed - ''' - return __virtualname__ if HAS_CONSUL else False + ''' + Only return if python-consul is installed + ''' + return __virtualname__ if HAS_CONSUL else False def ext_pillar(minion_id, pillar, # pylint: disable=W0613 conf): - ''' - Check consul for all data - ''' - comps = conf.split() + ''' + Check consul for all data + ''' + comps = conf.split() - profile = None - if comps[0]: - profile = comps[0] - client = get_conn(__opts__, profile) + profile = None + if comps[0]: + profile = comps[0] + client = get_conn(__opts__, profile) - path = '' - if len(comps) > 1 and comps[1].startswith('root='): - path = comps[1].replace('root=', '') + path = '' + if len(comps) > 1 and comps[1].startswith('root='): + path = comps[1].replace('root=', '') - # put the minion's ID in the path if necessary - path %= { - 'minion_id': minion_id - } + # put the minion's ID in the path if necessary + path %= { + 'minion_id': minion_id + } - try: - pillar = fetch_tree(client, path) - except KeyError: - log.error('No such key in consul profile {0}: {1}'.format(profile, path)) - pillar = {} + try: + pillar = fetch_tree(client, path) + except KeyError: + log.error('No such key in consul profile {0}: {1}' + .format(profile, path)) + pillar = {} + + return pillar - return pillar def consul_fetch(client, path): - ''' - Query consul for all keys/values within base path - ''' - return client.kv.get(path, recurse=True) + ''' + Query consul for all keys/values within base path + ''' + return client.kv.get(path, recurse=True) + def fetch_tree(client, path): - ''' - Grab data from consul, trim base path and remove any keys which - are folders. Take the remaining data and send it to be formatted - in such a way as to be used as pillar data. - ''' - index, items = consul_fetch(client, path) - ret = {} - has_children = re.compile(r'/$') - - log.debug('Fetched items: %r', format(items)) - - for item in reversed(items): - key = re.sub(r'^' + path + '/?', '', item['Key']) - if key != "": - log.debug('key/path - {0}: {1}'.format(path, key)) - log.debug('has_children? %r', format(has_children.search(key))) - if has_children.search(key) == None: - ret = pillar_format(ret, key.split('/'), item['Value']) - log.debug('Fetching subkeys for key: %r', format(item)) - - return ret + ''' + Grab data from consul, trim base path and remove any keys which + are folders. Take the remaining data and send it to be formatted + in such a way as to be used as pillar data. + ''' + index, items = consul_fetch(client, path) + ret = {} + has_children = re.compile(r'/$') + + log.debug('Fetched items: %r', format(items)) + + for item in reversed(items): + key = re.sub(r'^' + path + '/?', '', item['Key']) + if key != "": + log.debug('key/path - {0}: {1}'.format(path, key)) + log.debug('has_children? %r', format(has_children.search(key))) + if has_children.search(key) is None: + ret = pillar_format(ret, key.split('/'), item['Value']) + log.debug('Fetching subkeys for key: %r', format(item)) -def pillar_format(ret, keys, value): - ''' - Perform data formatting to be used as pillar data and - merge it with the current pillar data - ''' - if value == None: return ret - array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data - keyvalue = keys.pop() - pil = {keyvalue: pillar_value} - keys.reverse() - for k in keys: - pil = {k: pil} - return dict_merge(ret, pil) + +def pillar_format(ret, keys, value): + ''' + Perform data formatting to be used as pillar data and + merge it with the current pillar data + ''' + if value is None: + return ret + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data + keyvalue = keys.pop() + pil = {keyvalue: pillar_value} + keys.reverse() + for k in keys: + pil = {k: pil} + + return dict_merge(ret, pil) + def dict_merge(d1, d2): - ''' - Take 2 dictionaries and deep merge them - ''' - master = d1.copy() - for (k, v) in d2.iteritems(): - if k in master and isinstance(master[k], dict): - master[k] = dict_merge(master[k], v) - else: - master[k] = v - return master + ''' + Take 2 dictionaries and deep merge them + ''' + master = d1.copy() + for (k, v) in d2.iteritems(): + if k in master and isinstance(master[k], dict): + master[k] = dict_merge(master[k], v) + else: + master[k] = v + return master + def get_conn(opts, profile): - ''' - Return a client object for accessing consul - ''' - opts_pillar = opts.get('pillar', {}) - opts_master = opts_pillar.get('master', {}) - - opts_merged = {} - opts_merged.update(opts_master) - opts_merged.update(opts_pillar) - opts_merged.update(opts) - - if profile: - conf = opts_merged.get(profile, {}) - else: - conf = opts_merged - - consul_host = conf.get('consul.host', '127.0.0.1') - consul_port = conf.get('consul.port', 8500) - - if HAS_CONSUL: - return consul.Consul(host=consul_host, port=consul_port) - else: - raise CommandExecutionError( + ''' + Return a client object for accessing consul + ''' + opts_pillar = opts.get('pillar', {}) + opts_master = opts_pillar.get('master', {}) + + opts_merged = {} + opts_merged.update(opts_master) + opts_merged.update(opts_pillar) + opts_merged.update(opts) + + if profile: + conf = opts_merged.get(profile, {}) + else: + conf = opts_merged + + consul_host = conf.get('consul.host', '127.0.0.1') + consul_port = conf.get('consul.port', 8500) + + if HAS_CONSUL: + return consul.Consul(host=consul_host, port=consul_port) + else: + raise CommandExecutionError( '(unable to import consul, ' 'module most likely not installed. Download python-consul ' 'module and be sure to import consul)' - ) + ) From 66ecf6dade85f0b9021cc4dce45638c5a89cd31f Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Tue, 9 Jun 2015 17:05:38 +0100 Subject: [PATCH 368/769] Don't split multiline values into a list if wrapped inside " Use salt.utils.dictupdate to merge dictionaries --- src/saltext/consul/pillar/consul_pillar.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 6698224..df9b618 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -58,6 +58,7 @@ import re from salt.exceptions import CommandExecutionError +from salt.utils.dictupdate import update as dict_merge # Import third party libs try: @@ -149,8 +150,11 @@ def pillar_format(ret, keys, value): ''' if value is None: return ret - array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data + if value[0] == value[-1] == '"': + pillar_value = value[1:-1] + else: + array_data = value.split('\n') + pillar_value = array_data[0] if len(array_data) == 1 else array_data keyvalue = keys.pop() pil = {keyvalue: pillar_value} keys.reverse() @@ -160,19 +164,6 @@ def pillar_format(ret, keys, value): return dict_merge(ret, pil) -def dict_merge(d1, d2): - ''' - Take 2 dictionaries and deep merge them - ''' - master = d1.copy() - for (k, v) in d2.iteritems(): - if k in master and isinstance(master[k], dict): - master[k] = dict_merge(master[k], v) - else: - master[k] = v - return master - - def get_conn(opts, profile): ''' From 31ddfe0b64c8adb1ae5af7fb94ea4ad376c65044 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 17 Jun 2015 14:35:31 -0600 Subject: [PATCH 369/769] Fix extreme edge case where system encoding can not be detected --- src/saltext/consul/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index dc5bae4..80924e1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -47,8 +47,9 @@ def __define_global_system_encoding_variable__(): del locale if not encoding: # This is most likely asccii which is not the best but we were - # unable to find a better encoding - encoding = sys.getdefaultencoding() + # unable to find a better encoding. If this fails, we fall all + # the way back to ascii + encoding = sys.getdefaultencoding() or 'ascii' # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: From ea28b1e640aa8232c81a1dad7f02645c46430e8d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 16 Jun 2015 17:10:30 -0700 Subject: [PATCH 370/769] Changes to various modules to allow configuration items to be pulled via config.get using either module.key or module:key --- src/saltext/consul/modules/consul.py | 316 ++++++++------------------- 1 file changed, 96 insertions(+), 220 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 0cc42be..80ec9e2 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -29,6 +29,14 @@ __virtualname__ = 'consul' +def _get_config(): + ''' + Retrieve Consul configuration + ''' + return __salt__['config.get']('consul.url') or \ + __salt__['config.get']('consul:url') + + def _query(function, consul_url, method='GET', @@ -103,11 +111,8 @@ def list(consul_url=None, key=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -166,11 +171,8 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -229,11 +231,8 @@ def put(consul_url=None, key=None, value=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -337,11 +336,8 @@ def delete(consul_url=None, key=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -395,11 +391,8 @@ def agent_checks(consul_url=None): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -428,11 +421,8 @@ def agent_services(consul_url=None): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -462,11 +452,8 @@ def agent_members(consul_url=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -500,11 +487,8 @@ def agent_self(consul_url=None): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -542,11 +526,8 @@ def agent_maintenance(consul_url=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -596,11 +577,8 @@ def agent_join(consul_url=None, address=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -645,11 +623,8 @@ def agent_leave(consul_url=None, node=None): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -703,11 +678,8 @@ def agent_check_register(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -778,11 +750,8 @@ def agent_check_deregister(consul_url=None, checkid=None): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -826,11 +795,8 @@ def agent_check_pass(consul_url=None, checkid=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -878,11 +844,8 @@ def agent_check_warn(consul_url=None, checkid=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -930,11 +893,8 @@ def agent_check_fail(consul_url=None, checkid=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -997,11 +957,8 @@ def agent_service_register(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1080,11 +1037,8 @@ def agent_service_deregister(consul_url=None, serviceid=None): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1129,11 +1083,8 @@ def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1202,11 +1153,8 @@ def session_create(consul_url=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1281,11 +1229,8 @@ def session_list(consul_url=None, return_list=False, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1328,11 +1273,8 @@ def session_destroy(consul_url=None, session=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1378,11 +1320,8 @@ def session_info(consul_url=None, session=None, **kwargs): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1438,11 +1377,8 @@ def catalog_register(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1543,11 +1479,8 @@ def catalog_deregister(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1600,11 +1533,8 @@ def catalog_datacenters(consul_url=None): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1635,11 +1565,8 @@ def catalog_nodes(consul_url=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1674,11 +1601,8 @@ def catalog_services(consul_url=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1714,11 +1638,8 @@ def catalog_service(consul_url=None, service=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1760,11 +1681,8 @@ def catalog_node(consul_url=None, node=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1803,11 +1721,8 @@ def health_node(consul_url=None, node=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1846,11 +1761,8 @@ def health_checks(consul_url=None, service=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1894,11 +1806,8 @@ def health_service(consul_url=None, service=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1948,11 +1857,8 @@ def health_state(consul_url=None, state=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -1992,11 +1898,8 @@ def status_leader(consul_url=None): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2025,11 +1928,8 @@ def status_peers(consul_url): ''' ret = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2064,11 +1964,8 @@ def acl_create(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2123,11 +2020,8 @@ def acl_update(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2184,11 +2078,8 @@ def acl_delete(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2234,11 +2125,8 @@ def acl_info(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2276,11 +2164,8 @@ def acl_clone(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2324,11 +2209,8 @@ def acl_list(consul_url=None, **kwargs): ret = {} data = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2370,11 +2252,8 @@ def event_fire(consul_url=None, name=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False @@ -2430,11 +2309,8 @@ def event_list(consul_url=None, **kwargs): ret = {} query_params = {} if not consul_url: - try: - options = __salt__['config.option']('consul') - if not consul_url: - consul_url = options.get('url') - except (NameError, KeyError, AttributeError): + consul_url = _get_config() + if not consul_url: log.error('No Consul URL found.') ret['message'] = 'No Consul URL found.' ret['res'] = False From a950b254ded233990bff5d9cc382dbf872ca0c7e Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 3 Jul 2015 13:06:17 -0700 Subject: [PATCH 371/769] AsyncPillar wrapper to fix file_client: local minions Fixes #24440 --- src/saltext/consul/pillar/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 67eaf3d..257e331 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -68,8 +68,8 @@ def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs= saltenv = env ptype = { 'remote': AsyncRemotePillar, - #'local': AsyncPillar # TODO: implement - }.get(opts['file_client'], Pillar) + 'local': AsyncPillar, + }.get(opts['file_client'], AsyncPillar) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) @@ -688,3 +688,12 @@ def compile_pillar(self, ext=True, pillar_dirs=None): log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors return pillar + + +# TODO: actually migrate from Pillar to AsyncPillar to allow for futures in +# ext_pillar etc. +class AsyncPillar(Pillar): + @tornado.gen.coroutine + def compile_pillar(self, ext=True, pillar_dirs=None): + ret = super(AsyncPillar, self).compile_pillar(ext=ext, pillar_dirs=pillar_dirs) + raise tornado.gen.Return(ret) From d37e1380bd9eeb0fcc16564fba106f4c5e211d41 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 8 Jul 2015 16:26:04 +0300 Subject: [PATCH 372/769] Support pillarenv cmdline in state.sls. Backport of PR #23719 --- src/saltext/consul/pillar/__init__.py | 28 +++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8388a35..2c2756b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -28,7 +28,7 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, - pillar=None): + pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -45,7 +45,7 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, 'local': Pillar }.get(opts['file_client'], Pillar) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, - pillar=pillar) + pillar=pillar, pillarenv=pillarenv) class RemotePillar(object): @@ -53,13 +53,14 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, - pillar=None): + pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains self.id_ = id_ self.channel = salt.transport.Channel.factory(opts) + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -74,6 +75,7 @@ def compile_pillar(self): load = {'id': self.id_, 'grains': self.grains, 'saltenv': self.opts['environment'], + 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, 'ver': '2', 'cmd': '_pillar'} @@ -97,11 +99,11 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, - pillar=None): + pillar=None, pillarenv=None): # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, saltenv, ext) + self.opts = self.__gen_opts(opts, grains, id_, saltenv=saltenv, ext=ext, pillarenv=pillarenv) self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -145,7 +147,7 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): + def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' @@ -168,6 +170,8 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None): opts['id'] = id_ if 'environment' not in opts: opts['environment'] = saltenv + if 'pillarenv' not in opts: + opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): @@ -200,16 +204,16 @@ def get_tops(self): errors = [] # Gather initial top files try: - if self.opts['environment']: - tops[self.opts['environment']] = [ + if self.opts['pillarenv']: + tops[self.opts['pillarenv']] = [ compile_template( self.client.cache_file( self.opts['state_top'], - self.opts['environment'] + self.opts['pillarenv'] ), self.rend, self.opts['renderer'], - self.opts['environment'], + self.opts['pillarenv'], _pillar_rend=True ) ] @@ -343,8 +347,8 @@ def top_matches(self, top): ''' matches = {} for saltenv, body in top.items(): - if self.opts['environment']: - if saltenv != self.opts['environment']: + if self.opts['pillarenv']: + if saltenv != self.opts['pillarenv']: continue for match, data in body.items(): if self.matcher.confirm_top( From cef2236fd7b8372a26abe45d935e8ebd6404c609 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 8 Jul 2015 17:38:38 +0300 Subject: [PATCH 373/769] Minor bugfixes in pillarenv support --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 257e331..1569d95 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -246,7 +246,7 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pil opts['id'] = id_ if 'environment' not in opts: opts['environment'] = saltenv - if pillarenv not in opts: + if 'pillarenv' not in opts: opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] From e4314eb244135c2fd61473bce35fc19440fe994b Mon Sep 17 00:00:00 2001 From: Seth House Date: Fri, 17 Jul 2015 17:53:16 -0600 Subject: [PATCH 374/769] Add autosummary-generated docs --- docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/ref/pillar/all/salt.pillar.consul_pillar.rst diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst new file mode 100644 index 0000000..523ff0f --- /dev/null +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -0,0 +1,6 @@ +========================= +salt.pillar.consul_pillar +========================= + +.. automodule:: salt.pillar.consul_pillar + :members: \ No newline at end of file From 1558fc0870a4e0c58412c2d2600881a35d81901b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 21 Jul 2015 16:39:29 -0500 Subject: [PATCH 375/769] Support calling git_pillar using new config schema --- src/saltext/consul/pillar/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index cf8f4af..bd6a406 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -567,17 +567,10 @@ def render_pillar(self, matches): return pillar, errors - def _external_pillar_data(self, - pillar, - val, - pillar_dirs, - key): + def _external_pillar_data(self, pillar, val, pillar_dirs, key): ''' - Builds actual pillar data structure - and update - the variable ``pillar`` + Builds actual pillar data structure and updates the ``pillar`` variable ''' - ext = None # try the new interface, which includes the minion ID @@ -585,7 +578,14 @@ def _external_pillar_data(self, if isinstance(val, dict): ext = self.ext_pillars[key](self.opts['id'], pillar, **val) elif isinstance(val, list): - ext = self.ext_pillars[key](self.opts['id'], pillar, *val) + if key == 'git': + ext = self.ext_pillars[key](self.opts['id'], + val, + pillar_dirs) + else: + ext = self.ext_pillars[key](self.opts['id'], + pillar, + *val) else: if key == 'git': ext = self.ext_pillars[key](self.opts['id'], From 4874e74cbd06120c7d54da42602f11e10b571e1c Mon Sep 17 00:00:00 2001 From: Brad Thurber Date: Mon, 27 Jul 2015 15:38:05 -0400 Subject: [PATCH 376/769] PR to address issue #25496 add "modern" base64 encode/decode support --- src/saltext/consul/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 80924e1..e13a188 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -37,7 +37,7 @@ def __define_global_system_encoding_variable__(): # and reset to None encoding = sys.stdin.encoding if not encoding: - # If the system is properly codfigured this should return a valid + # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong # encoding import locale @@ -46,7 +46,7 @@ def __define_global_system_encoding_variable__(): # This is now garbage collectable del locale if not encoding: - # This is most likely asccii which is not the best but we were + # This is most likely ascii which is not the best but we were # unable to find a better encoding. If this fails, we fall all # the way back to ascii encoding = sys.getdefaultencoding() or 'ascii' From 245eadde0274755a8093a890139c864116a59431 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 6 Aug 2015 14:02:45 +0100 Subject: [PATCH 377/769] Don't stacktrace when trying to get the default locale. Fixes #26063 --- src/saltext/consul/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f9384f0..63cd6b1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -39,7 +39,12 @@ def __define_global_system_encoding_variable__(): # encoding. MS Windows has problems with this and reports the wrong # encoding import locale - encoding = locale.getdefaultlocale()[-1] + try: + encoding = locale.getdefaultlocale()[-1] + except ValueError: + # A bad locale setting was most likely found: + # https://github.com/saltstack/salt/issues/26063 + pass # This is now garbage collectable del locale From 1bbb377e8426857558bc66f3db50e8c2377c3f18 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 26 Aug 2015 23:26:06 -0500 Subject: [PATCH 378/769] salt/pillar/__init__.py: remove raw string formatting --- src/saltext/consul/pillar/__init__.py | 36 +++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 801404d..cbc4076 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -459,45 +459,53 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): - log.debug('Skipping ignored and missing SLS {0!r} in' - ' environment {1!r}'.format(sls, saltenv)) + log.debug('Skipping ignored and missing SLS \'{0}\' in' + ' environment \'{1}\''.format(sls, saltenv)) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): - msg = ('Specified SLS {0!r} in environment {1!r} is not' + msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not' ' available on the salt master').format(sls, saltenv) log.error(msg) errors.append(msg) else: - log.debug('Specified SLS {0!r} in environment {1!r} is not' - ' found, which might be due to environment {1!r}' + log.debug('Specified SLS \'{0}\' in environment \'{1}\' is not' + ' found, which might be due to environment \'{1}\'' ' not being present in "pillar_roots" yet!' .format(sls, saltenv)) # return state, mods, errors return None, mods, errors state = None try: - state = compile_template( - fn_, self.rend, self.opts['renderer'], saltenv, sls, _pillar_rend=True, **defaults) + state = compile_template(fn_, + self.rend, + self.opts['renderer'], + saltenv, + sls, + _pillar_rend=True, + **defaults) except Exception as exc: - msg = 'Rendering SLS {0!r} failed, render error:\n{1}'.format( + msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( sls, exc ) log.critical(msg) if self.opts.get('pillar_safe_render_error', True): - errors.append('Rendering SLS \'{0}\' failed. Please see master log for details.'.format(sls)) + errors.append( + 'Rendering SLS \'{0}\' failed. Please see master log for ' + 'details.'.format(sls) + ) else: errors.append(msg) mods.add(sls) nstate = None if state: if not isinstance(state, dict): - msg = 'SLS {0!r} does not render to a dictionary'.format(sls) + msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls) log.error(msg) errors.append(msg) else: if 'include' in state: if not isinstance(state['include'], list): - msg = ('Include Declaration in SLS {0!r} is not ' + msg = ('Include Declaration in SLS \'{0}\' is not ' 'formed as a list'.format(sls)) log.error(msg) errors.append(msg) @@ -550,12 +558,14 @@ def render_pillar(self, matches): if pstate is not None: if not isinstance(pstate, dict): log.error( - 'The rendered pillar sls file, {0!r} state did ' + 'The rendered pillar sls file, \'{0}\' state did ' 'not return the expected data format. This is ' 'a sign of a malformed pillar sls file. Returned ' 'errors: {1}'.format( sls, - ', '.join(['{0!r}'.format(e) for e in errors]) + ', '.join( + ['\'{0}\''.format(e) for e in errors] + ) ) ) continue From 89b41d0033e216bf4fb0f1b07d210b5f25c19fd4 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 3 Jul 2015 13:06:17 -0700 Subject: [PATCH 379/769] AsyncPillar wrapper to fix file_client: local minions Fixes #24440 --- src/saltext/consul/pillar/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bd6a406..801404d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -68,8 +68,8 @@ def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs= saltenv = env ptype = { 'remote': AsyncRemotePillar, - #'local': AsyncPillar # TODO: implement - }.get(opts['file_client'], Pillar) + 'local': AsyncPillar, + }.get(opts['file_client'], AsyncPillar) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) @@ -688,3 +688,12 @@ def compile_pillar(self, ext=True, pillar_dirs=None): log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors return pillar + + +# TODO: actually migrate from Pillar to AsyncPillar to allow for futures in +# ext_pillar etc. +class AsyncPillar(Pillar): + @tornado.gen.coroutine + def compile_pillar(self, ext=True, pillar_dirs=None): + ret = super(AsyncPillar, self).compile_pillar(ext=ext, pillar_dirs=pillar_dirs) + raise tornado.gen.Return(ret) From 25ced16e0b6b90a82d1bed8afce97193359d4ddb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 14 Sep 2015 15:39:17 -0600 Subject: [PATCH 380/769] Issue warning that some log levels may contain sensitive data --- src/saltext/consul/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index dd82a7d..14cb965 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -46,7 +46,7 @@ try: from salt.utils import parsers, ip_bracket from salt.utils.verify import check_user, verify_env, verify_socket - from salt.utils.verify import verify_files + from salt.utils.verify import verify_files, verify_log except ImportError as exc: if exc.args[0] != 'No module named _msgpack': raise @@ -113,6 +113,7 @@ def prepare(self): self.setup_logfile_logger() logger.info('Setting up the Salt Master') + verify_log(self.config) if self.config['transport'].lower() == 'zeromq': if not verify_socket(self.config['interface'], @@ -222,6 +223,7 @@ def prepare(self): self.config['id'] ) ) + verify_log(self.config) migrations.migrate_paths(self.config) if self.config['transport'].lower() == 'zeromq': # Late import so logging works correctly @@ -350,6 +352,7 @@ def prepare(self, proxydetails): self.config['id'] ) ) + verify_log(self.config) migrations.migrate_paths(self.config) # Late import so logging works correctly import salt.minion @@ -437,6 +440,7 @@ def prepare(self): self.config['id'] ) ) + verify_log(self.config) # Late import so logging works correctly import salt.minion From 81767604d4f3a3caf68ce52c6773c56172511a55 Mon Sep 17 00:00:00 2001 From: Bernard Kerckenaere Date: Thu, 17 Sep 2015 09:52:31 +0200 Subject: [PATCH 381/769] fix restart_on_error #27127 --- src/saltext/consul/pillar/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 801404d..77f33a4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -108,10 +108,14 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( - load, - dictkey='pillar', - ) + try: + ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( + load, + dictkey='pillar', + ) + except: + log.exception('Exception getting pillar:') + raise SaltClientError('Exception getting pillar.') if not isinstance(ret_pillar, dict): msg = ('Got a bad pillar from master, type {0}, expecting dict: ' From c6c60385a8a17bdb2e2309e9d62037ea02867111 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Tue, 22 Sep 2015 11:54:34 -0600 Subject: [PATCH 382/769] Updated module doc index using https://github.com/saltstack/salt/pull/27203 --- docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/ref/pillar/all/salt.pillar.consul_pillar.rst diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst new file mode 100644 index 0000000..f08d295 --- /dev/null +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -0,0 +1,6 @@ +salt.pillar.consul_pillar module +================================ + +.. automodule:: salt.pillar.consul_pillar + :members: + :undoc-members: From cd30516d49f49f56e79d366b2866f2fca0dc1371 Mon Sep 17 00:00:00 2001 From: Jonathan Ballet Date: Fri, 25 Sep 2015 16:21:46 +0200 Subject: [PATCH 383/769] doc: fixed indentation in salt.modules.consul's documentation --- src/saltext/consul/modules/consul.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 80ec9e2..547df37 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -160,13 +160,15 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): salt '*' consul.list key='web' recurse='True' decode='True' - By default values stored in Consul are base64 encoded, passing the - decode option will show them as the decoded values. + By default values stored in Consul are base64 encoded, passing the + decode option will show them as the decoded values. + + .. code-block:: bash salt '*' consul.list key='web' recurse='True' decode='True' raw='True' - By default Consult will return other information about the key, the raw - option will return only the raw value. + By default Consult will return other information about the key, the raw + option will return only the raw value. ''' ret = {} From 2749144a78caf43e3462519e505652c3a7df64b7 Mon Sep 17 00:00:00 2001 From: Jonathan Ballet Date: Fri, 25 Sep 2015 16:21:46 +0200 Subject: [PATCH 384/769] doc: fixed indentation in salt.modules.consul's documentation --- src/saltext/consul/modules/consul.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 80ec9e2..547df37 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -160,13 +160,15 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): salt '*' consul.list key='web' recurse='True' decode='True' - By default values stored in Consul are base64 encoded, passing the - decode option will show them as the decoded values. + By default values stored in Consul are base64 encoded, passing the + decode option will show them as the decoded values. + + .. code-block:: bash salt '*' consul.list key='web' recurse='True' decode='True' raw='True' - By default Consult will return other information about the key, the raw - option will return only the raw value. + By default Consult will return other information about the key, the raw + option will return only the raw value. ''' ret = {} From eee7473cb92cc56f6cf583e50997e486f89c1261 Mon Sep 17 00:00:00 2001 From: Matthias Erll Date: Wed, 28 Oct 2015 13:56:44 +0100 Subject: [PATCH 385/769] Consider each pillar match only once. --- src/saltext/consul/pillar/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 77f33a4..d7c3633 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -446,10 +446,12 @@ def top_matches(self, top): self.opts.get('nodegroups', {}), ): if saltenv not in matches: - matches[saltenv] = [] + matches[saltenv] = env_matches = [] + else: + env_matches = matches[saltenv] for item in data: - if isinstance(item, six.string_types): - matches[saltenv].append(item) + if isinstance(item, six.string_types) and item not in env_matches: + env_matches.append(item) return matches def render_pstate(self, sls, saltenv, mods, defaults=None): From 57542c8a163244c0d25b46ed192d5aea76b3cc3f Mon Sep 17 00:00:00 2001 From: Matthias Erll Date: Wed, 28 Oct 2015 16:27:04 +0100 Subject: [PATCH 386/769] Do not merge previous values in pillar include loop. --- src/saltext/consul/pillar/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d7c3633..84e1635 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -522,17 +522,17 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): mods, defaults ) - if nstate: - if key: - nstate = { - key: nstate - } - - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get('renderer', 'yaml')) + if nstate: + if key: + nstate = { + key: nstate + } + + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml')) if err: errors += err From 86860898bbf285d682a63a0e56620590c04cd3ad Mon Sep 17 00:00:00 2001 From: Matthias Erll Date: Thu, 29 Oct 2015 07:27:03 +0100 Subject: [PATCH 387/769] Variable err is from previous loop, too. --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 84e1635..f8f1df0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -534,8 +534,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): self.merge_strategy, self.opts.get('renderer', 'yaml')) - if err: - errors += err + if err: + errors += err return state, mods, errors def render_pillar(self, matches): From 7190355c90eb4df91c3beb73d07515451ab1e20a Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Tue, 3 Nov 2015 17:43:10 +0100 Subject: [PATCH 388/769] Fix typos of parameter --- src/saltext/consul/modules/consul.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 547df37..8b0c312 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -2032,7 +2032,7 @@ def acl_update(consul_url=None, **kwargs): if 'id' in kwargs: data['ID'] = kwargs['id'] else: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2088,7 +2088,7 @@ def acl_delete(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2135,7 +2135,7 @@ def acl_info(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2174,7 +2174,7 @@ def acl_clone(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2219,7 +2219,7 @@ def acl_list(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret From 68d698c8f04452dd4a6460f2dc9264a32b87d1f6 Mon Sep 17 00:00:00 2001 From: William Huba Date: Mon, 9 Nov 2015 15:29:11 -0500 Subject: [PATCH 389/769] Make consul.list a function alias. Fixes #28712. --- src/saltext/consul/modules/consul.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 80ec9e2..0df837a 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -26,6 +26,11 @@ from salt.exceptions import SaltInvocationError +# Don't shadow built-ins. +__func_alias__ = { + 'list_': 'list' +} + __virtualname__ = 'consul' @@ -92,7 +97,7 @@ def _query(function, return ret -def list(consul_url=None, key=None, **kwargs): +def list_(consul_url=None, key=None, **kwargs): ''' List keys in Consul From f757dc7a4dba18fac81a5a55b0979a4019708dd5 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sun, 29 Nov 2015 10:03:24 -0800 Subject: [PATCH 390/769] various fixes to the consul execution module, in particular a fix to address #29119 --- src/saltext/consul/modules/consul.py | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index c1d07da..a82c7ad 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -68,6 +68,10 @@ def _query(function, base_url = _urljoin(consul_url, '{0}/'.format(api_version)) url = _urljoin(base_url, function, False) + if data is None: + data = {} + data = json.dumps(data) + result = salt.utils.http.query( url, method=method, @@ -309,7 +313,7 @@ def put(consul_url=None, key=None, value=None, **kwargs): ret = _query(consul_url=consul_url, function=function, method=method, - data=json.dumps(data), + data=data, query_params=query_params) if ret['res']: @@ -695,9 +699,7 @@ def agent_check_register(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] else: - ret['message'] = 'Required parameter "name" is missing.' - ret['res'] = False - return ret + raise SaltInvocationError('Required argument "name" is missing.') if True not in [True for item in ('script', 'http') if item in kwargs]: ret['message'] = 'Required parameter "script" or "http" is missing.' @@ -973,6 +975,8 @@ def agent_service_register(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'address' in kwargs: data['Address'] = kwargs['address'] @@ -1031,7 +1035,7 @@ def agent_service_deregister(consul_url=None, serviceid=None): Used to remove a service. :param consul_url: The Consul server URL. - :param name: A name describing the service. + :param serviceid: A serviceid describing the service. :return: Boolean and message indicating success or failure. CLI Example: @@ -1176,6 +1180,8 @@ def session_create(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'checks' in kwargs: data['Touch'] = kwargs['touch'] @@ -1455,11 +1461,11 @@ def catalog_register(consul_url=None, **kwargs): if res['res']: ret['res'] = True ret['message'] = ('Catalog registration ' - 'for {0} successful.'.format(kwargs['name'])) + 'for {0} successful.'.format(kwargs['node'])) else: ret['res'] = False ret['message'] = ('Catalog registration ' - 'for {0} failed.'.format(kwargs['name'])) + 'for {0} failed.'.format(kwargs['node'])) return ret @@ -1516,11 +1522,11 @@ def catalog_deregister(consul_url=None, **kwargs): data=data) if res['res']: ret['res'] = True - ret['message'] = 'Catalog item {0} removed.'.format(kwargs['name']) + ret['message'] = 'Catalog item {0} removed.'.format(kwargs['node']) else: ret['res'] = False ret['message'] = ('Removing Catalog ' - 'item {0} failed.'.format(kwargs['name'])) + 'item {0} failed.'.format(kwargs['node'])) return ret @@ -1980,6 +1986,8 @@ def acl_create(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2043,6 +2051,8 @@ def acl_update(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2325,6 +2335,8 @@ def event_list(consul_url=None, **kwargs): if 'name' in kwargs: query_params = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') function = 'event/list/' ret = _query(consul_url=consul_url, From 425c714525ce7ec535552a91953c9f5c05017137 Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Thu, 3 Dec 2015 17:18:26 +0000 Subject: [PATCH 391/769] Update consul_pillar.py In regards to issue #29193 --- src/saltext/consul/pillar/consul_pillar.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index df9b618..aa8c839 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -131,6 +131,9 @@ def fetch_tree(client, path): log.debug('Fetched items: %r', format(items)) + if items is None: + return ret + for item in reversed(items): key = re.sub(r'^' + path + '/?', '', item['Key']) if key != "": From 05eddf15b82d353d6a854b36d41404a9880fbf2d Mon Sep 17 00:00:00 2001 From: Brett Mack Date: Thu, 3 Dec 2015 18:27:53 +0000 Subject: [PATCH 392/769] Removed extra line --- src/saltext/consul/pillar/consul_pillar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index aa8c839..b84b91c 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -133,7 +133,6 @@ def fetch_tree(client, path): if items is None: return ret - for item in reversed(items): key = re.sub(r'^' + path + '/?', '', item['Key']) if key != "": From 707a580d7366bd82d06dc1f4d7ffb3e11c64b5d4 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 18 Dec 2015 12:30:49 -0700 Subject: [PATCH 393/769] Add exclude_ext_pillar option --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4134843..493f5d0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -626,6 +626,8 @@ def ext_pillar(self, pillar, pillar_dirs): # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: + if run in self.opts['exclude_ext_pillar']: + continue if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} From ca78170d76b53651bc7c8a67ae73a6ce1f24ae0d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 22 Dec 2015 09:12:37 -0700 Subject: [PATCH 394/769] Doh! --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 493f5d0..72eb1c2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -626,7 +626,7 @@ def ext_pillar(self, pillar, pillar_dirs): # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: - if run in self.opts['exclude_ext_pillar']: + if run in self.opts.get('exclude_ext_pillar'): continue if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') From 87e2f010b4146c28ccc0cb11b4e80ec1cdc0bec1 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 22 Dec 2015 09:14:53 -0700 Subject: [PATCH 395/769] use a sane default --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 72eb1c2..5f4d909 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -626,7 +626,7 @@ def ext_pillar(self, pillar, pillar_dirs): # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: - if run in self.opts.get('exclude_ext_pillar'): + if run in self.opts.get('exclude_ext_pillar', []): continue if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') From 321c11dff3b184c35754c33d0cb5f9a8f5aa1ccb Mon Sep 17 00:00:00 2001 From: Sean Jenkins Date: Tue, 29 Dec 2015 10:22:54 -0700 Subject: [PATCH 396/769] Remove recurse_list from pillar_source_merging_strategy and add pillar_merge_list (bool) instead --- src/saltext/consul/pillar/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5f4d909..95dd159 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -540,7 +540,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state, nstate, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) if err: errors += err @@ -579,7 +580,8 @@ def render_pillar(self, matches): pillar, pstate, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) return pillar, errors @@ -667,7 +669,8 @@ def ext_pillar(self, pillar, pillar_dirs): pillar, ext, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) ext = None return pillar @@ -684,7 +687,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): pillar = merge(pillar, self.opts['pillar'], self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 5a6188de4b6b50c45459d0df2f89fb63ba6240c6 Mon Sep 17 00:00:00 2001 From: Sean Jenkins Date: Wed, 6 Jan 2016 09:55:20 -0700 Subject: [PATCH 397/769] Set (pillar_)merge_lists to default for PR#30062 --- src/saltext/consul/pillar/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 95dd159..b35185a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -541,7 +541,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) if err: errors += err @@ -581,7 +581,7 @@ def render_pillar(self, matches): pstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) return pillar, errors @@ -670,7 +670,7 @@ def ext_pillar(self, pillar, pillar_dirs): ext, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) ext = None return pillar @@ -688,7 +688,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'], self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 49db1aa27cee935433f7f3546a6c3bda492dac10 Mon Sep 17 00:00:00 2001 From: Andreas Lutro Date: Thu, 14 Jan 2016 14:18:11 +0100 Subject: [PATCH 398/769] make sure *all* ignore_missing slses are catched previously, only the last statement in top.sls with `ignore_missing: true` would actually be effective. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f8f1df0..4e97fd8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -395,7 +395,7 @@ def merge_tops(self, tops): if isinstance(comp, six.string_types): states[comp] = True if ignore_missing: - self.ignored_pillars[saltenv] = list(states.keys()) + self.ignored_pillars[saltenv].extend(states.keys()) top[saltenv][tgt] = matches top[saltenv][tgt].extend(states) return self.sort_top_targets(top, orders) From d5d4f671f3a4506b7cbee92cb8c01578f697f05c Mon Sep 17 00:00:00 2001 From: Sean Jenkins Date: Tue, 29 Dec 2015 10:22:54 -0700 Subject: [PATCH 399/769] Remove recurse_list from pillar_source_merging_strategy and add pillar_merge_list (bool) instead --- src/saltext/consul/pillar/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4e97fd8..26b8d52 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -532,7 +532,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state, nstate, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) if err: errors += err @@ -569,7 +570,8 @@ def render_pillar(self, matches): pillar, pstate, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) return pillar, errors @@ -655,7 +657,8 @@ def ext_pillar(self, pillar, pillar_dirs): pillar, ext, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) ext = None return pillar @@ -672,7 +675,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): pillar = merge(pillar, self.opts['pillar'], self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', 'True')) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 1a5b6bbf6a1806b69e717b107adaa24dcea9595f Mon Sep 17 00:00:00 2001 From: Sean Jenkins Date: Wed, 6 Jan 2016 09:55:20 -0700 Subject: [PATCH 400/769] Set (pillar_)merge_lists to default for PR#30062 --- src/saltext/consul/pillar/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 26b8d52..7b905e1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -533,7 +533,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) if err: errors += err @@ -571,7 +571,7 @@ def render_pillar(self, matches): pstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) return pillar, errors @@ -658,7 +658,7 @@ def ext_pillar(self, pillar, pillar_dirs): ext, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) ext = None return pillar @@ -676,7 +676,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'], self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'True')) + self.opts.get('pillar_merge_lists', 'False')) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From d06417c1a7c42edcaa6bc96875bb96575a4c7a9b Mon Sep 17 00:00:00 2001 From: Andreas Lutro Date: Thu, 21 Jan 2016 09:11:04 +0100 Subject: [PATCH 401/769] fix KeyError when adding ignored pillars --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4e97fd8..711e91e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -395,6 +395,8 @@ def merge_tops(self, tops): if isinstance(comp, six.string_types): states[comp] = True if ignore_missing: + if saltenv not in self.ignored_pillars: + self.ignored_pillars[saltenv] = [] self.ignored_pillars[saltenv].extend(states.keys()) top[saltenv][tgt] = matches top[saltenv][tgt].extend(states) From c50275c781b6212a4778793f06014df9805694cf Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Jan 2016 01:04:51 +0000 Subject: [PATCH 402/769] Stop the deprecation path started in 0.17 about ext_pillar The ``ext_pillar`` functions **must** now accept a minion ID as the first argument. This stops the deprecation path started in Salt 0.17.x. Before this minion ID first argument was introduced, the minion ID could be retrieved accessing ``__opts__['id']`` loosing the reference to the master ID initially set in opts. This is no longer the case, ``__opts__['id']`` will be kept as the master ID. --- src/saltext/consul/pillar/__init__.py | 41 +++++++++------------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0e46cd2..d56ffbf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -247,7 +247,6 @@ def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pil opts['grains'] = {} else: opts['grains'] = grains - opts['id'] = id_ if 'environment' not in opts: opts['environment'] = saltenv if 'pillarenv' not in opts: @@ -635,35 +634,23 @@ def ext_pillar(self, pillar, pillar_dirs): return {} for key, val in six.iteritems(run): if key not in self.ext_pillars: - err = ('Specified ext_pillar interface {0} is ' - 'unavailable').format(key) - log.critical(err) + log.critical( + 'Specified ext_pillar interface {0} is ' + 'unavailable'.format(key) + ) continue try: - try: - ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) - except TypeError as exc: - if str(exc).startswith('ext_pillar() takes exactly '): - log.warning('Deprecation warning: ext_pillar "{0}"' - ' needs to accept minion_id as first' - ' argument'.format(key)) - else: - raise - - ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) - except Exception as exc: + ext = self._external_pillar_data(pillar, + val, + pillar_dirs, + key) + except Exception as exc: # pylint: disable=broad-except log.exception( - 'Failed to load ext_pillar {0}: {1}'.format( - key, - exc - ) - ) + 'Failed to load ext_pillar {0}: {1}'.format( + key, + exc + ) + ) if ext: pillar = merge( pillar, From 0499d0063c8cf7bacf7ac61a0f89464f615e0173 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Jan 2016 12:52:06 +0000 Subject: [PATCH 403/769] Be explicit to which id we're referring to. Actually pass the minion ID. --- src/saltext/consul/pillar/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d56ffbf..413d290 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -176,12 +176,13 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): + self.minion_id = minion_id # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -227,7 +228,7 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pillarenv=None): + def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' @@ -316,7 +317,7 @@ def get_tops(self): errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) - log.error('Pillar rendering failed for minion {0}: '.format(self.opts['id']), + log.error('Pillar rendering failed for minion {0}: '.format(self.minion_id), exc_info=True) # Search initial top files for includes @@ -590,26 +591,24 @@ def _external_pillar_data(self, pillar, val, pillar_dirs, key): ''' ext = None - # try the new interface, which includes the minion ID - # as first argument if isinstance(val, dict): - ext = self.ext_pillars[key](self.opts['id'], pillar, **val) + ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): if key == 'git': - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, val, pillar_dirs) else: - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, pillar, *val) else: if key == 'git': - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, val, pillar_dirs) else: - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, pillar, val) return ext From d059d6edda2470c0d518bc229ba673fb4ebcf2ab Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Jan 2016 13:42:30 +0000 Subject: [PATCH 404/769] Pass the "minion_opts" where needed. --- src/saltext/consul/pillar/__init__.py | 31 +++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 413d290..2a71025 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) -def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, +def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option @@ -48,12 +48,12 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, 'remote': RemotePillar, 'local': Pillar }.get(opts['file_client'], Pillar) - return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) # TODO: migrate everyone to this one! -def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, +def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option @@ -70,7 +70,7 @@ def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs= 'remote': AsyncRemotePillar, 'local': AsyncPillar, }.get(opts['file_client'], AsyncPillar) - return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) @@ -78,13 +78,13 @@ class AsyncRemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains - self.id_ = id_ + self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) self.opts['pillarenv'] = pillarenv self.pillar_override = {} @@ -99,7 +99,7 @@ def compile_pillar(self): ''' Return a future which will contain the pillar data from the master ''' - load = {'id': self.id_, + load = {'id': self.minion_id, 'grains': self.grains, 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], @@ -130,13 +130,13 @@ class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains - self.id_ = id_ + self.minion_id = minion_id self.channel = salt.transport.Channel.factory(opts) self.opts['pillarenv'] = pillarenv self.pillar_override = {} @@ -150,7 +150,7 @@ def compile_pillar(self): ''' Return the pillar data from the master ''' - load = {'id': self.id_, + load = {'id': self.minion_id, 'grains': self.grains, 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], @@ -183,7 +183,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) - self.client = salt.fileclient.get_file_client(self.opts, True) + minion_opts = copy.deepcopy(self.opts) + minion_opts['id'] = minion_id + self.minion_opts = minion_opts + self.client = salt.fileclient.get_file_client(self.minion_opts, True) if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -194,12 +197,12 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts, utils=utils) else: - self.functions = salt.loader.minion_mods(self.opts, utils=utils) + self.functions = salt.loader.minion_mods(self.minion_opts, utils=utils) else: self.functions = functions - self.matcher = salt.minion.Matcher(self.opts, self.functions) - self.rend = salt.loader.render(self.opts, self.functions) + self.matcher = salt.minion.Matcher(self.minion_opts, self.functions) + self.rend = salt.loader.render(self.minion_opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 ext_pillar_opts = dict(self.opts) From 10b632a27df89f4d23228131cb7646c48019d050 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Jan 2016 14:29:37 +0000 Subject: [PATCH 405/769] Keep old opts "behavior" and be explicit on which options dict is used --- src/saltext/consul/pillar/__init__.py | 45 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 2a71025..0397693 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -176,40 +176,46 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + def __init__(self, opts, grains, minion_id, saltenv, ext=None, + functions=None, pillar=None, pillarenv=None): + # Keep a copy of the incoming master opts. Any changes made to it won't change + # the passed dictionary, they are local to this class + self.master_opts = copy.deepcopy(opts) self.minion_id = minion_id # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) - minion_opts = copy.deepcopy(self.opts) - minion_opts['id'] = minion_id - self.minion_opts = minion_opts - self.client = salt.fileclient.get_file_client(self.minion_opts, True) + self.opts = self.__gen_opts(self.master_opts, + grains, + saltenv=saltenv, + ext=ext, + pillarenv=pillarenv) + self.client = salt.fileclient.get_file_client(self.opts, True) - if opts.get('file_client', '') == 'local': - opts['grains'] = grains + if self.master_opts.get('file_client', '') == 'local': + self.master_opts['grains'] = grains # if we didn't pass in functions, lets load them if functions is None: - utils = salt.loader.utils(opts) - if opts.get('file_client', '') == 'local': - self.functions = salt.loader.minion_mods(opts, utils=utils) + utils = salt.loader.utils(self.master_opts) + if self.master_opts.get('file_client', '') == 'local': + self.functions = salt.loader.minion_mods(self.master_opts, utils=utils) else: - self.functions = salt.loader.minion_mods(self.minion_opts, utils=utils) + self.functions = salt.loader.minion_mods(self.opts, utils=utils) else: self.functions = functions - self.matcher = salt.minion.Matcher(self.minion_opts, self.functions) - self.rend = salt.loader.render(self.minion_opts, self.functions) + self.matcher = salt.minion.Matcher(self.opts, self.functions) + self.rend = salt.loader.render(self.opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 - ext_pillar_opts = dict(self.opts) + # Additionally, pass the master options in order not to loose + # the master id under __opts__ + ext_pillar_opts = copy.deepcopy(self.master_opts) ext_pillar_opts['file_roots'] = self.actual_file_roots self.merge_strategy = 'smart' - if opts.get('pillar_source_merging_strategy'): - self.merge_strategy = opts['pillar_source_merging_strategy'] + if self.master_opts.get('pillar_source_merging_strategy'): + self.merge_strategy = self.master_opts['pillar_source_merging_strategy'] self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} @@ -244,7 +250,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren ) # Backwards compatibility saltenv = env - opts = dict(opts_in) + opts = copy.deepcopy(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' if not grains: @@ -266,6 +272,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren opts['ext_pillar'].append(ext) else: opts['ext_pillar'] = [ext] + opts['id'] = self.minion_id return opts def _get_envs(self): From c6930c1297d975ffd261a8d4614dd1333cd39ae6 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Jan 2016 16:36:52 +0000 Subject: [PATCH 406/769] Revert "Keep old opts "behavior" and be explicit on which options dict is used" This reverts commit 10b632a27df89f4d23228131cb7646c48019d050. --- src/saltext/consul/pillar/__init__.py | 45 +++++++++++---------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0397693..2a71025 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -176,46 +176,40 @@ class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, minion_id, saltenv, ext=None, - functions=None, pillar=None, pillarenv=None): - # Keep a copy of the incoming master opts. Any changes made to it won't change - # the passed dictionary, they are local to this class - self.master_opts = copy.deepcopy(opts) + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, + pillar=None, pillarenv=None): self.minion_id = minion_id # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(self.master_opts, - grains, - saltenv=saltenv, - ext=ext, - pillarenv=pillarenv) - self.client = salt.fileclient.get_file_client(self.opts, True) + self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + minion_opts = copy.deepcopy(self.opts) + minion_opts['id'] = minion_id + self.minion_opts = minion_opts + self.client = salt.fileclient.get_file_client(self.minion_opts, True) - if self.master_opts.get('file_client', '') == 'local': - self.master_opts['grains'] = grains + if opts.get('file_client', '') == 'local': + opts['grains'] = grains # if we didn't pass in functions, lets load them if functions is None: - utils = salt.loader.utils(self.master_opts) - if self.master_opts.get('file_client', '') == 'local': - self.functions = salt.loader.minion_mods(self.master_opts, utils=utils) + utils = salt.loader.utils(opts) + if opts.get('file_client', '') == 'local': + self.functions = salt.loader.minion_mods(opts, utils=utils) else: - self.functions = salt.loader.minion_mods(self.opts, utils=utils) + self.functions = salt.loader.minion_mods(self.minion_opts, utils=utils) else: self.functions = functions - self.matcher = salt.minion.Matcher(self.opts, self.functions) - self.rend = salt.loader.render(self.opts, self.functions) + self.matcher = salt.minion.Matcher(self.minion_opts, self.functions) + self.rend = salt.loader.render(self.minion_opts, self.functions) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 - # Additionally, pass the master options in order not to loose - # the master id under __opts__ - ext_pillar_opts = copy.deepcopy(self.master_opts) + ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots self.merge_strategy = 'smart' - if self.master_opts.get('pillar_source_merging_strategy'): - self.merge_strategy = self.master_opts['pillar_source_merging_strategy'] + if opts.get('pillar_source_merging_strategy'): + self.merge_strategy = opts['pillar_source_merging_strategy'] self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} @@ -250,7 +244,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren ) # Backwards compatibility saltenv = env - opts = copy.deepcopy(opts_in) + opts = dict(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' if not grains: @@ -272,7 +266,6 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren opts['ext_pillar'].append(ext) else: opts['ext_pillar'] = [ext] - opts['id'] = self.minion_id return opts def _get_envs(self): From 2eab1265fb46aa47cf41c86fd38d44a6250c6d7b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Jan 2016 17:09:45 +0000 Subject: [PATCH 407/769] Keep the old side effect behavior --- src/saltext/consul/pillar/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 2a71025..a6faeb2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -183,10 +183,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) - minion_opts = copy.deepcopy(self.opts) - minion_opts['id'] = minion_id - self.minion_opts = minion_opts - self.client = salt.fileclient.get_file_client(self.minion_opts, True) + self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -197,16 +194,18 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if opts.get('file_client', '') == 'local': self.functions = salt.loader.minion_mods(opts, utils=utils) else: - self.functions = salt.loader.minion_mods(self.minion_opts, utils=utils) + self.functions = salt.loader.minion_mods(self.opts, utils=utils) else: self.functions = functions - self.matcher = salt.minion.Matcher(self.minion_opts, self.functions) - self.rend = salt.loader.render(self.minion_opts, self.functions) + self.matcher = salt.minion.Matcher(self.opts, self.functions) + self.rend = salt.loader.render(self.opts, self.functions) + ext_pillar_opts = copy.deepcopy(self.opts) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 - ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots + # Keep the incoming opts ID intact, ie, the master id + ext_pillar_opts['id'] = opts['id'] self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -244,7 +243,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren ) # Backwards compatibility saltenv = env - opts = dict(opts_in) + opts = copy.deepcopy(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' if not grains: @@ -253,6 +252,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren opts['grains'] = grains if 'environment' not in opts: opts['environment'] = saltenv + opts['id'] = self.minion_id if 'pillarenv' not in opts: opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): From 071d472e871e5684b0291e48f6d3fdeb810db6a5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 Jan 2016 18:01:06 +0000 Subject: [PATCH 408/769] The ID might not be set on the master --- src/saltext/consul/pillar/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a6faeb2..89a02a2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -205,7 +205,8 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, # location of file_roots. Issue 5951 ext_pillar_opts['file_roots'] = self.actual_file_roots # Keep the incoming opts ID intact, ie, the master id - ext_pillar_opts['id'] = opts['id'] + if 'id' in opts: + ext_pillar_opts['id'] = opts['id'] self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] From e7514831535078115a7784c7b8fbb88bf3024a9d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 27 Jan 2016 10:31:02 -0600 Subject: [PATCH 409/769] Add additional reason for pillar env being found --- src/saltext/consul/pillar/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 9bbcc66..4692cf5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -476,10 +476,15 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: - log.debug('Specified SLS {0!r} in environment {1!r} is not' - ' found, which might be due to environment {1!r}' - ' not being present in "pillar_roots" yet!' - .format(sls, saltenv)) + log.debug( + 'Specified SLS \'%s\' in environment \'%s\' was not ' + 'found. This could be because SLS \'%s\' is in an ' + 'environment other than \'%s\', but \'%s\' is included in ' + 'that environment\'s Pillar top file. It could also be ' + 'due to environment \'%s\' not being defined in ' + '"pillar_roots"', + sls, saltenv, sls, saltenv, saltenv, saltenv + ) # return state, mods, errors return None, mods, errors state = None From 08b0a3542f1191a49bb8fe93e9c0a4934d6f482d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 25 Jan 2016 11:43:16 -0600 Subject: [PATCH 410/769] Pillar cache for master Entry point for pillar cache Working msgpack implementation along with in-memory Remove debugging Perms for pillar cache creation Documentation and small typo fix Additional documentation note in master config A note on pillar caching in the documentation on scaling Backport of pillar cache to 2015.8 branch Fixed ext pillar with debug Lint --- src/saltext/consul/pillar/__init__.py | 100 +++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4692cf5..8484fb9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -9,6 +9,7 @@ import os import collections import logging +import tornado.gen # Import salt libs import salt.loader @@ -17,6 +18,7 @@ import salt.crypt import salt.transport import salt.utils.url +import salt.utils.cache from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -26,8 +28,6 @@ # Import 3rd-party libs import salt.ext.six as six -import tornado.gen - log = logging.getLogger(__name__) @@ -48,6 +48,13 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, 'remote': RemotePillar, 'local': Pillar }.get(opts['file_client'], Pillar) + # If local pillar and we're caching, run through the cache system first + log.info('Determining pillar cache') + if opts['pillar_cache']: + log.info('Compiling pillar from cache') + log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) + return PillarCache(opts, grains, id_, saltenv, ext=ext, functions=funcs, + pillar=pillar, pillarenv=pillarenv) return ptype(opts, grains, id_, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) @@ -172,6 +179,95 @@ def compile_pillar(self): return ret_pillar +class PillarCache(object): + ''' + Return a cached pillar if it exists, otherwise cache it. + + Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs. + Each saltenv contains a pillar dict + + Example data structure: + + ``` + {'minion_1': + {'base': {'pilar_key_1' 'pillar_val_1'} + } + ''' + # TODO ABC? + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, + pillar=None, pillarenv=None): + # Yes, we need all of these because we need to route to the Pillar object + # if we have no cache. This is another refactor target. + + # Go ahead and assign these because they may be needed later + self.opts = opts + self.grains = grains + self.minion_id = minion_id + self.ext = ext + self.functions = functions + self.pillar = pillar + self.pillarenv = pillarenv + + if saltenv is None: + self.saltenv = 'base' + else: + self.saltenv = saltenv + + # Determine caching backend + self.cache = salt.utils.cache.CacheFactory.factory( + self.opts['pillar_cache_backend'], + self.opts['pillar_cache_ttl'], + minion_cache_path=self._minion_cache_path(minion_id)) + + def _minion_cache_path(self, minion_id): + ''' + Return the path to the cache file for the minion. + + Used only for disk-based backends + ''' + return os.path.join(self.opts['cachedir'], 'pillar_cache', minion_id) + + def fetch_pillar(self): + ''' + In the event of a cache miss, we need to incur the overhead of caching + a new pillar. + ''' + log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) + fresh_pillar = Pillar(self.opts, + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar=self.pillar, + pillarenv=self.pillarenv) + return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here + + def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs + log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + log.debug('Scanning cache: {0}'.format(self.cache._dict)) + # Check the cache! + if self.minion_id in self.cache: # Keyed by minion_id + # TODO Compare grains, etc? + if self.saltenv in self.cache[self.minion_id]: + # We have a cache hit! Send it back. + log.debug('Pillar cache hit for minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + return self.cache[self.minion_id][self.saltenv] + else: + # We found the minion but not the env. Store it. + fresh_pillar = self.fetch_pillar() + self.cache[self.minion_id][self.saltenv] = fresh_pillar + log.debug('Pillar cache miss for saltenv {0} for minion {1}'.format(self.saltenv, self.minion_id)) + return fresh_pillar + else: + # We haven't seen this minion yet in the cache. Store it. + fresh_pillar = self.fetch_pillar() + self.cache[self.minion_id] = {self.saltenv: fresh_pillar} + log.debug('Pillar cache miss for minion {0}'.format(self.minion_id)) + log.debug('Current pillar cache: {0}'.format(self.cache._dict)) # FIXME hack! + return fresh_pillar + + class Pillar(object): ''' Read over the pillar top files and render the pillar data From e27c252cabdc17b1294084db72674fd75b27ef41 Mon Sep 17 00:00:00 2001 From: Andreas Lutro Date: Tue, 2 Feb 2016 15:50:20 +0100 Subject: [PATCH 411/769] Properly set the default value for pillar_merge_lists Issue: #30809 Related PRs: #30062 #30458 --- src/saltext/consul/pillar/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8484fb9..fe30942 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -636,7 +636,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'False')) + self.opts.get('pillar_merge_lists', False)) if err: errors += err @@ -674,7 +674,7 @@ def render_pillar(self, matches): pstate, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'False')) + self.opts.get('pillar_merge_lists', False)) return pillar, errors @@ -761,7 +761,7 @@ def ext_pillar(self, pillar, pillar_dirs): ext, self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'False')) + self.opts.get('pillar_merge_lists', False)) ext = None return pillar @@ -779,7 +779,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'], self.merge_strategy, self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', 'False')) + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From b7cfe2ddc5d56ecc848aa321155a9e28a72a0c3d Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Wed, 3 Feb 2016 12:46:02 -0700 Subject: [PATCH 412/769] Fix external pillar excludes Refs #29846 --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ef8594e..efb9639 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -637,11 +637,11 @@ def ext_pillar(self, pillar, pillar_dirs): # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: - if run in self.opts.get('exclude_ext_pillar', []): - continue if not isinstance(run, dict): log.critical('The "ext_pillar" option is malformed') return {} + if run.keys()[0] in self.opts.get('exclude_ext_pillar', []): + continue for key, val in six.iteritems(run): if key not in self.ext_pillars: log.critical( From e01c67da61b6718a090f0b573014c4d1d8adda4a Mon Sep 17 00:00:00 2001 From: ketzacoatl Date: Fri, 5 Feb 2016 02:30:33 -0500 Subject: [PATCH 413/769] update consul ext_pillar to support root based on role If there is a "role" grain, provide that as an option to format the path prefix. --- src/saltext/consul/pillar/consul_pillar.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index b84b91c..223ebd5 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -32,13 +32,14 @@ - consul: my_consul_config - consul: my_other_consul_config -The ``minion_id`` may be used in the ``root`` path to expose minion-specific -information stored in consul. +Either the ``minion_id``, or the ``role`` grain may be used in the ``root`` +path to expose minion-specific information stored in consul. .. code-block:: yaml ext_pillar: - consul: my_consul_config root=/salt/%(minion_id)s + - consul: my_consul_config root=/salt/%(role)s Minion-specific values may override shared values when the minion-specific root appears after the shared root: @@ -49,6 +50,13 @@ - consul: my_consul_config root=/salt-shared - consul: my_other_consul_config root=/salt-private/%(minion_id)s +If using the ``role`` grain in the consul key path, be sure to define it using +`/etc/salt/grains`, or similar: + +.. code-block:: yaml + + role: my-minion-role + ''' from __future__ import absolute_import @@ -97,9 +105,11 @@ def ext_pillar(minion_id, if len(comps) > 1 and comps[1].startswith('root='): path = comps[1].replace('root=', '') + role = __salt__['grains.get']('role') # put the minion's ID in the path if necessary path %= { - 'minion_id': minion_id + 'minion_id': minion_id, + 'role': role } try: From 1ba155f63be0c9d10a1fc53c9a67825e62951edf Mon Sep 17 00:00:00 2001 From: ketzacoatl Date: Fri, 5 Feb 2016 02:39:31 -0500 Subject: [PATCH 414/769] update key formatting/handling in consul ext_pillar If the value of the key we've looked up is in the form: - foo - bar treat it as a list, and drop the '- ', otherwise, leave it be. --- src/saltext/consul/pillar/consul_pillar.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index b84b91c..30d1370 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -150,13 +150,20 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' + # skip it if value is None: return ret + # if wrapped in quotes, drop them if value[0] == value[-1] == '"': pillar_value = value[1:-1] - else: + # if we have a list, reformat into a list + if value[0] == '-' and value[1] == ' ': array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data + # drop the '- ' on each element + pillar_value = [elem[2:] for elem in array_data] + # leave it be + else: + pillar_value = value keyvalue = keys.pop() pil = {keyvalue: pillar_value} keys.reverse() From 351b5e20aec53756e575b6535613c3a4040d60ed Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 10 Feb 2016 15:45:28 -0700 Subject: [PATCH 415/769] Change all relevant Boron references to 2016.3.0 And more Boron deprecation warnings to Carbon --- src/saltext/consul/pillar/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index efb9639..b4fffc1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -38,9 +38,9 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs= ''' if env is not None: salt.utils.warn_until( - 'Boron', + 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Boron.' + 'not \'env\'. This functionality will be removed in Salt Carbon.' ) # Backwards compatibility saltenv = env @@ -60,9 +60,9 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, ''' if env is not None: salt.utils.warn_until( - 'Boron', + 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Boron.' + 'not \'env\'. This functionality will be removed in Salt Carbon.' ) # Backwards compatibility saltenv = env @@ -237,10 +237,10 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren ''' if env is not None: salt.utils.warn_until( - 'Boron', + 'Carbon', 'Passing a salt environment should be done using \'saltenv\' ' 'not \'env\'. This functionality will be removed in Salt ' - 'Boron.' + 'Carbon.' ) # Backwards compatibility saltenv = env From dfc235e5ac0fda66945f9aeb45688b045477b82a Mon Sep 17 00:00:00 2001 From: ketzacoatl Date: Fri, 12 Feb 2016 11:24:03 -0500 Subject: [PATCH 416/769] strip whitespace from pillar keys looked up in consul of all types, and from both sides (leading / trailing) --- src/saltext/consul/pillar/consul_pillar.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index aeebd67..e8899a5 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -160,6 +160,8 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' + # drop leading/trailing whitespace, if any + value = value.strip(' \t\n\r') # skip it if value is None: return ret From 5fd1a3728416bb4b200b836478dd35c9ff35fb4b Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Fri, 12 Feb 2016 18:04:33 -0500 Subject: [PATCH 417/769] Full control over client options in Consul pillar --- src/saltext/consul/pillar/consul_pillar.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index e8899a5..e4d706f 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -12,6 +12,13 @@ my_consul_config: consul.host: 127.0.0.1 consul.port: 8500 + consul.token: b6376760-a8bb-edd5-fcda-33bc13bfc556 + consul.scheme: http + consul.consistency: default + consul.dc: dev + consul.verify: True + +All parameters are optional. After the profile is created, configure the external pillar system to use it. Optionally, a root may be specified. @@ -203,11 +210,14 @@ def get_conn(opts, profile): else: conf = opts_merged - consul_host = conf.get('consul.host', '127.0.0.1') - consul_port = conf.get('consul.port', 8500) + params = {} + for key in ('host', 'port', 'token', 'scheme', 'consistency', 'dc', 'verify'): + prefixed_key = 'consul.{key}'.format(key=key) + if prefixed_key in conf: + params[key] = conf[prefixed_key] if HAS_CONSUL: - return consul.Consul(host=consul_host, port=consul_port) + return consul.Consul(**params) else: raise CommandExecutionError( '(unable to import consul, ' From 6095b608d4979c8dbbda2d376ae1193727034c03 Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Mon, 15 Feb 2016 00:47:47 -0500 Subject: [PATCH 418/769] Consul SDB driver A simple SDB driver for Consul. Allows full control over all client options. --- src/saltext/consul/sdb/consul.py | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/saltext/consul/sdb/consul.py diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py new file mode 100644 index 0000000..4750164 --- /dev/null +++ b/src/saltext/consul/sdb/consul.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +''' +Consul sdb Module + +:maintainer: SaltStack +:maturity: New +:platform: all + +This module allows access to Consul using an ``sdb://`` URI + +Like all sdb modules, the Consul module requires a configuration profile to +be configured in either the minion or master configuration file. This profile +requires very little. For example: + +.. code-block:: yaml + myconsul: + driver: consul + host: 127.0.0.1 + port: 8500 + token: b6376760-a8bb-edd5-fcda-33bc13bfc556 + scheme: http + consistency: default + dc: dev + verify: True + +The ``driver`` refers to the Consul module, all other options are optional. +For option details see: http://python-consul.readthedocs.org/en/latest/#consul +''' +from __future__ import absolute_import + +from salt.exceptions import CommandExecutionError + +try: + import consul + HAS_CONSUL = True +except ImportError: + HAS_CONSUL = False + + +__func_alias__ = { + 'set_': 'set' +} + + +def set_(key, value, profile=None): + if not profile: + return False + + conn = get_conn(profile) + + return conn.kv.put(key, value) + + +def get(key, profile=None): + if not profile: + return False + + conn = get_conn(profile) + + _, result = conn.kv.get(key) + + return result['Value'] if result else None + + +def get_conn(profile): + ''' + Return a client object for accessing consul + ''' + params = {} + for key in ('host', 'port', 'token', 'scheme', 'consistency', 'dc', 'verify'): + if key in profile: + params[key] = profile[key] + + if HAS_CONSUL: + return consul.Consul(**params) + else: + raise CommandExecutionError( + '(unable to import consul, ' + 'module most likely not installed. PLease install python-consul)' + ) From cd4e1f01e8d7a274d8a7776d85b50e515dcdefb6 Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Tue, 16 Feb 2016 17:52:39 -0500 Subject: [PATCH 419/769] Consul Pillar: select DC based on pillarenv This PR provides functionality that allows mapping pillarenv values to Consul datacenters --- src/saltext/consul/pillar/consul_pillar.py | 92 ++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index e4d706f..cd36261 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -20,6 +20,47 @@ All parameters are optional. +If you have a multi-datacenter Consul cluster you can map your ``pillarenv``s +to your data centers by providing a dictionary of mappings in ``consul.dc`` +field: + +.. code-block:: yaml + + my_consul_config: + consul.dc: + dev: us-east-1 + prod: us-west-1 + +In the example above we specifying static mapping between Pillar environments +and data centers: the data for ``dev`` and ``prod`` Pillar environments will +be fetched from ``us-east-1`` and ``us-west-1`` datacenter respectively. + +In fact when ``consul.dc`` is set to dictionary keys are processed as regular +expressions (that can capture named parameters) and values are processed as +string templates as per PEP 3101. + +.. code-block:: yaml + + my_consul_config: + consul.dc: + ^dev-.*$: dev-datacenter + ^(?P.*)-prod$: prod-datacenter-{region} + +This example maps all Pillar environments starting with ``dev-`` to +``dev-datacenter`` whereas Pillar environment like ``eu-prod`` will be +mapped to ``prod-datacenter-eu``. + +Before evaluation patterns are sorted by length in descending order. + +If Pillar environment names correspond to data center names a single pattern +can be used: + +.. code-block:: yaml + + my_consul_config: + consul.dc: + ^(?P.*)$: '{env}' + After the profile is created, configure the external pillar system to use it. Optionally, a root may be specified. @@ -216,6 +257,10 @@ def get_conn(opts, profile): if prefixed_key in conf: params[key] = conf[prefixed_key] + if 'dc' in params: + pillarenv = opts_merged.get('pillarenv') or 'base' + params['dc'] = _resolve_datacenter(params['dc'], pillarenv) + if HAS_CONSUL: return consul.Consul(**params) else: @@ -224,3 +269,50 @@ def get_conn(opts, profile): 'module most likely not installed. Download python-consul ' 'module and be sure to import consul)' ) + + +def _resolve_datacenter(dc, pillarenv): + ''' + If ``dc`` is a string - return it as is. + + If it's a dict then sort it in descending order by key length and try + to use keys as RegEx patterns to match against ``pillarenv``. + The value for matched pattern should be a string (that can use + ``str.format`` syntax togetehr with captured variables from pattern) + pointing to targe data center to use. + + If none patterns matched return ``None`` which meanse us datacenter of + conencted Consul agent. + ''' + log.debug("Resolving Consul datacenter based on: {dc}".format(dc=dc)) + + try: + mappings = dc.items() # is it a dict? + except AttributeError: + log.debug('Using pre-defined DC: {dc!r}'.format(dc=dc)) + return dc + + log.debug( + 'Selecting DC based on pillarenv using {num} pattern(s)'.format( + num=len(mappings) + ) + ) + log.debug('Pillarenv set to {env!r}'.format(env=pillarenv)) + + # sort in reverse based on pattern length + # but use alphabetic order within groups of patterns of same length + sorted_mappings = sorted(mappings, key=lambda m: (-len(m[0]), m[0])) + + for pattern, target in sorted_mappings: + match = re.match(pattern, pillarenv) + if match: + log.debug('Matched pattern: {pattern!r}'.format(pattern=pattern)) + result = target.format(**match.groupdict()) + log.debug('Resolved datacenter: {result!r}'.format(result=result)) + return result + + log.debug( + 'None of following patterns matched pillarenv={env}: {lst}'.format( + env=pillarenv, lst=', '.join(repr(x) for x in mappings) + ) + ) From 01ea9b91973d7df672c0b009d199f496fdf9e114 Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Tue, 9 Feb 2016 12:28:25 -0700 Subject: [PATCH 420/769] salt/pillar/__init__.py: remove env support --- src/saltext/consul/pillar/__init__.py | 31 +++------------------------ 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7fcfc99..94bb6e1 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -31,19 +31,11 @@ log = logging.getLogger(__name__) -def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs=None, +def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' - if env is not None: - salt.utils.warn_until( - 'Carbon', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Carbon.' - ) - # Backwards compatibility - saltenv = env ptype = { 'remote': RemotePillar, 'local': Pillar @@ -60,19 +52,11 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs= # TODO: migrate everyone to this one! -def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs=None, +def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' - if env is not None: - salt.utils.warn_until( - 'Carbon', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Carbon.' - ) - # Backwards compatibility - saltenv = env ptype = { 'remote': AsyncRemotePillar, 'local': AsyncPillar, @@ -327,19 +311,10 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillarenv=None): + def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' - if env is not None: - salt.utils.warn_until( - 'Carbon', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt ' - 'Carbon.' - ) - # Backwards compatibility - saltenv = env opts = copy.deepcopy(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' From 02dd8946dbdb0d4e2e978ff8d85c69893fa89810 Mon Sep 17 00:00:00 2001 From: Kris Raney Date: Thu, 18 Feb 2016 09:41:37 -0600 Subject: [PATCH 421/769] Let ext_pillar_first determine the override order --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7fcfc99..1e1f1b8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -777,8 +777,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = merge(pillar, - self.opts['pillar'], + pillar = merge(self.opts['pillar'], + pillar, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) From 3ce86362e83932836dc0cc129276e3d24f5d72de Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 18 Feb 2016 19:24:34 +0300 Subject: [PATCH 422/769] Use current pillarenv if nothing specified with argument ... instead of overwriting it with 'None' and getting merged pillar environments. This is actual at least for pillar.items call that should return actual pillar data for the minion, but currently it returns merged data from all available environments. --- src/saltext/consul/pillar/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7fcfc99..ba9b84a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -93,7 +93,8 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.grains = grains self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) - self.opts['pillarenv'] = pillarenv + if pillarenv is not None or 'pillarenv' not in self.opts: + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -145,7 +146,8 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.grains = grains self.minion_id = minion_id self.channel = salt.transport.Channel.factory(opts) - self.opts['pillarenv'] = pillarenv + if pillarenv is not None or 'pillarenv' not in self.opts: + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): From 42cdc8cc180dc5c625e2cc5cc1b0a37f4cdeddab Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sat, 20 Feb 2016 14:39:01 +0100 Subject: [PATCH 423/769] Bring up ext_pillar rendering errors as well --- src/saltext/consul/pillar/__init__.py | 38 ++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fe30942..66c071b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -642,13 +642,14 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): errors += err return state, mods, errors - def render_pillar(self, matches): + def render_pillar(self, matches, errors=None): ''' Extract the sls pillar files from the matches and render them into the pillar ''' pillar = copy.copy(self.pillar_override) - errors = [] + if errors is None: + errors = [] for saltenv, pstates in six.iteritems(matches): mods = set() for sls in pstates: @@ -708,22 +709,26 @@ def _external_pillar_data(self, pillar, val, pillar_dirs, key): val) return ext - def ext_pillar(self, pillar, pillar_dirs): + def ext_pillar(self, pillar, pillar_dirs, errors=None): ''' Render the external pillar data ''' + if errors is None: + errors = [] if 'ext_pillar' not in self.opts: - return pillar + return pillar, errors if not isinstance(self.opts['ext_pillar'], list): - log.critical('The "ext_pillar" option is malformed') - return pillar + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return pillar, errors ext = None # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: if not isinstance(run, dict): - log.critical('The "ext_pillar" option is malformed') - return {} + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return {}, errors for key, val in six.iteritems(run): if key not in self.ext_pillars: err = ('Specified ext_pillar interface {0} is ' @@ -749,12 +754,8 @@ def ext_pillar(self, pillar, pillar_dirs): pillar_dirs, key) except Exception as exc: - log.exception( - 'Failed to load ext_pillar {0}: {1}'.format( - key, - exc - ) - ) + errors.append('Failed to load ext_pillar {0}: {1}'.format( + key, exc)) if ext: pillar = merge( pillar, @@ -763,7 +764,7 @@ def ext_pillar(self, pillar, pillar_dirs): self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) ext = None - return pillar + return pillar, errors def compile_pillar(self, ext=True, pillar_dirs=None): ''' @@ -772,9 +773,9 @@ def compile_pillar(self, ext=True, pillar_dirs=None): top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) + pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge(pillar, self.opts['pillar'], self.merge_strategy, @@ -783,7 +784,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = self.ext_pillar(pillar, pillar_dirs) + pillar, errors = self.ext_pillar( + pillar, pillar_dirs, errors=errors) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 330b1d209d5d1d9d3c390ca801f7bde95607fa81 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sat, 20 Feb 2016 14:39:01 +0100 Subject: [PATCH 424/769] Bring up ext_pillar rendering errors as well --- src/saltext/consul/pillar/__init__.py | 40 ++++++++++++++------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7fcfc99..b8e5642 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -654,13 +654,14 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): errors += err return state, mods, errors - def render_pillar(self, matches): + def render_pillar(self, matches, errors=None): ''' Extract the sls pillar files from the matches and render them into the pillar ''' pillar = copy.copy(self.pillar_override) - errors = [] + if errors is None: + errors = [] for saltenv, pstates in six.iteritems(matches): mods = set() for sls in pstates: @@ -720,22 +721,26 @@ def _external_pillar_data(self, pillar, val, pillar_dirs, key): val) return ext - def ext_pillar(self, pillar, pillar_dirs): + def ext_pillar(self, pillar, pillar_dirs, errors=None): ''' Render the external pillar data ''' + if errors is None: + errors = [] if 'ext_pillar' not in self.opts: - return pillar + return pillar, errors if not isinstance(self.opts['ext_pillar'], list): - log.critical('The "ext_pillar" option is malformed') - return pillar + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return pillar, errors ext = None # Bring in CLI pillar data pillar.update(self.pillar_override) for run in self.opts['ext_pillar']: if not isinstance(run, dict): - log.critical('The "ext_pillar" option is malformed') - return {} + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return {}, errors if run.keys()[0] in self.opts.get('exclude_ext_pillar', []): continue for key, val in six.iteritems(run): @@ -750,13 +755,9 @@ def ext_pillar(self, pillar, pillar_dirs): val, pillar_dirs, key) - except Exception as exc: # pylint: disable=broad-except - log.exception( - 'Failed to load ext_pillar {0}: {1}'.format( - key, - exc - ) - ) + except Exception as exc: + errors.append('Failed to load ext_pillar {0}: {1}'.format( + key, exc)) if ext: pillar = merge( pillar, @@ -765,7 +766,7 @@ def ext_pillar(self, pillar, pillar_dirs): self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) ext = None - return pillar + return pillar, errors def compile_pillar(self, ext=True, pillar_dirs=None): ''' @@ -774,9 +775,9 @@ def compile_pillar(self, ext=True, pillar_dirs=None): top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) + pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge(pillar, self.opts['pillar'], self.merge_strategy, @@ -785,7 +786,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = self.ext_pillar(pillar, pillar_dirs) + pillar, errors = self.ext_pillar( + pillar, pillar_dirs, errors=errors) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 698b6a5c2be43ccfbfea6f3743bfb7a59f73b9f2 Mon Sep 17 00:00:00 2001 From: Kris Raney Date: Thu, 25 Feb 2016 10:17:39 -0600 Subject: [PATCH 425/769] Add pillar_roots_override_ext_pillar option and deprecate ext_pillar_first --- src/saltext/consul/pillar/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 1e1f1b8..fa8b6ed 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -773,15 +773,22 @@ def compile_pillar(self, ext=True, pillar_dirs=None): ''' top, top_errors = self.get_top() if ext: - if self.opts.get('ext_pillar_first', False): + if self.opts.get('pillar_roots_override_ext_pillar', False) or self.opts.get('ext_pillar_first', False): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = merge(self.opts['pillar'], - pillar, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + if self.opts.get('pillar_roots_override_ext_pillar', False): + pillar = merge(self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + else: + pillar = merge(pillar, + self.opts['pillar'], + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 94213997ed92edfe8f7e4973667e0e654fbe2ac2 Mon Sep 17 00:00:00 2001 From: Kris Raney Date: Thu, 25 Feb 2016 11:02:06 -0600 Subject: [PATCH 426/769] Fix pylint error and add a deprecation warning --- src/saltext/consul/pillar/__init__.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fa8b6ed..b99c0fa 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -773,22 +773,27 @@ def compile_pillar(self, ext=True, pillar_dirs=None): ''' top, top_errors = self.get_top() if ext: + if self.opts.get('ext_pillar_first', False): + salt.utils.warn_until('Nitrogen', + 'The \'ext_pillar_first\' option has been deprecated and ' + 'replaced by \'pillar_roots_override_ext_pillar\'.' + ) if self.opts.get('pillar_roots_override_ext_pillar', False) or self.opts.get('ext_pillar_first', False): self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) if self.opts.get('pillar_roots_override_ext_pillar', False): - pillar = merge(self.opts['pillar'], - pillar, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge(self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: - pillar = merge(pillar, - self.opts['pillar'], - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge(pillar, + self.opts['pillar'], + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From e0799789a4bf17aa00cf4ef3d999f97187923ef2 Mon Sep 17 00:00:00 2001 From: Kris Raney Date: Fri, 26 Feb 2016 11:49:36 -0600 Subject: [PATCH 427/769] Fix merge error --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 1e5457d..d10d04b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -756,7 +756,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): 'The \'ext_pillar_first\' option has been deprecated and ' 'replaced by \'pillar_roots_override_ext_pillar\'.' ) - self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) if self.opts.get('pillar_roots_override_ext_pillar', False): From 653c400670695e1195d07e5888433b6529a9d859 Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Sat, 27 Feb 2016 12:45:19 -0500 Subject: [PATCH 428/769] Consul Pillar: don't crash on empty values --- src/saltext/consul/pillar/consul_pillar.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index cd36261..af15f78 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -208,11 +208,13 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' - # drop leading/trailing whitespace, if any - value = value.strip(' \t\n\r') - # skip it + # if value is empty in Consul then it's None here - skip it if value is None: return ret + + # drop leading/trailing whitespace, if any + value = value.strip(' \t\n\r') + # if wrapped in quotes, drop them if value[0] == value[-1] == '"': pillar_value = value[1:-1] From 85e9598e51c6dcc08d1410e43e2ffd4e5e503306 Mon Sep 17 00:00:00 2001 From: Borys Pierov Date: Sat, 27 Feb 2016 13:09:17 -0500 Subject: [PATCH 429/769] Use YAML to parse values in Consul pillar --- src/saltext/consul/pillar/consul_pillar.py | 23 ++++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index af15f78..3a421c2 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Use consul data as a Pillar source +Use Consul K/V as a Pillar source with values parsed as YAML :depends: - python-consul @@ -113,6 +113,8 @@ import re +import yaml + from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge @@ -212,20 +214,11 @@ def pillar_format(ret, keys, value): if value is None: return ret - # drop leading/trailing whitespace, if any - value = value.strip(' \t\n\r') - - # if wrapped in quotes, drop them - if value[0] == value[-1] == '"': - pillar_value = value[1:-1] - # if we have a list, reformat into a list - if value[0] == '-' and value[1] == ' ': - array_data = value.split('\n') - # drop the '- ' on each element - pillar_value = [elem[2:] for elem in array_data] - # leave it be - else: - pillar_value = value + # If value is not None then it's a string + # Use YAML to parse the data + # YAML strips whitespaces unless they're surrounded by quotes + pillar_value = yaml.load(value) + keyvalue = keys.pop() pil = {keyvalue: pillar_value} keys.reverse() From 1203b3a066a3af99686d1d91a904ea06aeee5507 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 7 Mar 2016 21:36:30 -0600 Subject: [PATCH 430/769] Fall back to False when pillar_opts not set This should have been changed when we altered the default value in 16a3938. It caused a missing pillar_opts minion config value to incorrectly assume that master config should have been added to the pillar data. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 2c2756b..2922a5f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -586,7 +586,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) errors.extend(top_errors) - if self.opts.get('pillar_opts', True): + if self.opts.get('pillar_opts', False): mopts = dict(self.opts) if 'grains' in mopts: mopts.pop('grains') From b607b616fe6664df6fa98e5b352b347d448c9687 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Mon, 14 Mar 2016 16:34:22 -0600 Subject: [PATCH 431/769] Updated release notes Added a shortcut to link to pull requests Added doc entries for all new modules removed the unnecessary :undoc-members: option from recent doc module entries --- docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst index f08d295..c27449b 100644 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -3,4 +3,3 @@ salt.pillar.consul_pillar module .. automodule:: salt.pillar.consul_pillar :members: - :undoc-members: From 14a59555f4ec8ae67830672ddd3aaa1c6430dd9a Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 15 Mar 2016 14:55:35 -0600 Subject: [PATCH 432/769] Pylint fixes --- src/saltext/consul/__init__.py | 2 +- src/saltext/consul/modules/consul.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 2e90399..f47130c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -60,7 +60,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins # pylint: disable=incompatible-py3-code + import __builtin__ as builtins else: import builtins # pylint: disable=import-error diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index a82c7ad..4bbf229 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1199,7 +1199,7 @@ def session_create(consul_url=None, **kwargs): if str(_ttl).endswith('s'): _ttl = _ttl[:-1] - if not int(_ttl) >= 0 and not int(_ttl) <= 3600: + if int(_ttl) < 0 or int(_ttl) > 3600: ret['message'] = ('TTL must be ', 'between 0 and 3600.') ret['res'] = False From 2be9ddfda0ba96170ceae81d04eff57d68027043 Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Fri, 18 Mar 2016 17:37:22 -0600 Subject: [PATCH 433/769] Fixed build errors, additional mock imports, new module docs Fixes #31975 --- docs/ref/sdb/all/salt.sdb.consul.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/ref/sdb/all/salt.sdb.consul.rst diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst new file mode 100644 index 0000000..5a47d2b --- /dev/null +++ b/docs/ref/sdb/all/salt.sdb.consul.rst @@ -0,0 +1,5 @@ +salt.sdb.consul module +====================== + +.. automodule:: salt.sdb.consul + :members: From b0ff48559d62eeec4d666c3bdb0b323441b4f068 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 31 Mar 2016 17:31:32 -0500 Subject: [PATCH 434/769] use dictupdate.merge instead of dict.update to merge CLI pillar overrides Fixes #18429. --- src/saltext/consul/pillar/__init__.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 567c06a..46493aa 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -723,7 +723,13 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): return pillar, errors ext = None # Bring in CLI pillar data - pillar.update(self.pillar_override) + if self.pillar_override and isinstance(self.pillar_override, dict): + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + for run in self.opts['ext_pillar']: if not isinstance(run, dict): errors.append('The "ext_pillar" option is malformed') @@ -802,6 +808,14 @@ def compile_pillar(self, ext=True, pillar_dirs=None): for error in errors: log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors + + if self.pillar_override and isinstance(self.pillar_override, dict): + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + return pillar From d9bc7eaf87b687fd3c69181c8767ad3f7bec411b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 7 Apr 2016 10:10:53 -0500 Subject: [PATCH 435/769] Improve git_pillar documentation/logging * Add note about different behavior of top file in git_pillar * Make log entry for a missing pillar SLS file more accurate for git_pillar --- src/saltext/consul/pillar/__init__.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 46493aa..ae70cb9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -572,15 +572,24 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: - log.debug( - 'Specified SLS \'%s\' in environment \'%s\' was not ' - 'found. This could be because SLS \'%s\' is in an ' - 'environment other than \'%s\', but \'%s\' is included in ' - 'that environment\'s Pillar top file. It could also be ' - 'due to environment \'%s\' not being defined in ' - '"pillar_roots"', - sls, saltenv, sls, saltenv, saltenv, saltenv - ) + msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' + 'found. '.format(sls, saltenv)) + if self.opts.get('__git_pillar', False) is True: + msg += ( + 'This is likely caused by a git_pillar top file ' + 'containing an environment other than the one for the ' + 'branch in which it resides. Each git_pillar ' + 'branch/tag must have its own top file.' + ) + else: + msg += ( + 'This could be because SLS \'{0}\' is in an ' + 'environment other than \'{1}\', but \'{1}\' is ' + 'included in that environment\'s Pillar top file. It ' + 'could also be due to environment \'{1}\' not being ' + 'defined in \'pillar_roots\'.'.format(sls, saltenv) + ) + log.debug(msg) # return state, mods, errors return None, mods, errors state = None From 7ade93e6be5be80b47cebc884fdc08e3f211ed2b Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Mon, 11 Apr 2016 17:07:15 -0600 Subject: [PATCH 436/769] [develop] Merge forward from 2016.3 to develop (#32494) * fix sorting by latest version when called with an attribute * remove reference to master_alive_check * Fixes saltstack/salt#28262 * Resolve memory leak in authentication * outputter virt_list does not exist anymore * Update proxmox documentation * Fix documentation on boto_asg and boto_elb modules and states * modules.win_timezone: don't list all zones in debug log * Correcty index glusterfs bricks Fixes issue #32311 * Cleaner deprecation process with decorators * Add deprecation decorator scaffold * Capture type error and unhandled exceptions while function calls * Aware of the current and future version of deprecation * Implement initially is_deprecated decorator * Add an alias for the capitalization * Fix capitalization easier way * Remove an extra line * Add successor name to the deprecation decorator. * Granulate logging and error messages. * Implement function swapper * Raise later the caught exception * Clarify exception message * Save function original name * Remove an extra line * Hide an alternative hidden function name in the error message, preserving the error itself * Rename variable as private * Add a method to detect if a function is using its previous version * Message to the log and/or raise an exception accordingly to the status of used function * Log an error along with the exception * Add internal method documentation * Add documentation and usage process for decorator "is_deprecated" * Add documentation and process usage for the decorator "with_deprecated" * Hide private method name * Fix PEP8, re-word the error message * Deprecate basic uptime function * Add initial decorator unit test * Rename old/new functions, mock versions * Move frequent data to the test setup * Add logging on EOL exception * Rename and document high to low version test on is_deprecated * Implement a test on low to high version of is_deprecated decorator * Add a correction to the test description * Remove a dead code * Implement a test for high to low version on is_deprecated, using with_successor param * Correct typso adn mistaeks * Implement high to low version with successor param on is_deprecated * Setup a virtual name for the module * Implement test for with_deprecated should raise an exception if same deprecated function not found * Implement test for with_deprecated an old function is picked up if configured * Correct test description purpose * Implement test with_deprecated when no deprecation is requested * Add logging test to the configured deprecation request * Add logging testing when deprecated version wasn't requested * Implement test EOL for with_deprecated decorator * Correct test explanation * Rename the test * Implement with_deprecated no EOL, deprecated other function name * Implement with_deprecated, deprecated other function name, EOL reached * Add test description for the with_deprecated + with_name + EOL * Fix confusing test names * Add logging test to the is_deprecated decorator when function as not found. * Add more test point to each test, remove empty lines * Bugfix: at certain conditions a wrong alias name is reported to the log * Fix a typo in a comment * Add test for the logging * Disable a pylint: None will _never_ be raised * Fix test for the deprecated "status.uptime" version * Bugfix: Do not yank raised exceptions * Remove unnecessary decorator * Add test for the new uptime * Add test for the new uptime fails when /proc/uptime does not exists * Rename old test case * Skip test for the UTC time, unless freeze time is used. * Fix pylint * Fix documentation * Bugfix: proxy-pass the docstring of the decorated function * Lint fix * Fixes saltstack/salt#28262 for 2015.5 branch * Update master config docs * Improve git_pillar documentation/logging * Add note about different behavior of top file in git_pillar * Make log entry for a missing pillar SLS file more accurate for git_pillar * FreeBSD supports packages in format java/openjdk7 so the prior commit broke that functionality. Check freebsd/pkg#1409 for more info. * FreeBSD supports packages in format java/openjdk7 so the prior commit broke that functionality. Check freebsd/pkg#1409 for more info. * Update glusterfs_test to be inline with #32312 * Fix salt-cloud paralell provisioning Closes #31632 * Ignore Raspbian in service.py __virtual__ (#32421) * Ignore Raspbian in service.py __virtual__ This prevents more than one execution module from trying to load as the service virtual module. Refs: #32413 * pack __salt__ before loading provider overrides We can (and should) pack here since we're just packing a reference to the object. __salt__ needs to be available when we're loading our provider overrides * Fix broken __salt__ dict in provider override Using ret.items() here sets ``__salt__`` to its items (tuple containing function name and reference), breaking usage of ``__salt__`` inside overridden functions. * Merge #32293 with test fixes (#32418) * Fix issue #11497 * Remove check for working directory presence in tests * Fix Domainname introspection Default value needs to be extracted from the container itself, because dockerd set Domainname value when network_mode=host. * Add pgjsonb_queue to queue doc index * Pylint fixes * Pass parser options into batch mode Resolves #31738 * Changed the target file in file.symlink test (#32443) * Argument name in docs should match actual arg name (#32445) Fixes #31851 * tests.integration: bypass MacOS TMPDIR, gettempdir (#32447) Updates 0edd532, 8f558a5. When logging in as root over `ssh root@host`, `$TMPDIR` and `tempfile.gettempdir()` are both set to a variation of: ``` /private/var/folders/zz/zyxvpxvq6csfxvn_n0000000000000/T/ ``` When logging in as root over `sudo -i`, `$TMPDIR` is unset and `tempfile.gettempdir()` is set to `/tmp`. My guess is that the second case is an unintended or uncorrected omision by Apple as they have introduced the longer, randomized temp path in a recent version of MacOS. * Issue #28706: Fix state user.present behavior. (#32448) - As mentionned in issue #28706, state user.present no longer remove user from groups if the keyword 'groups' with empty value '[]' is not explicitly set, salt will assume current groups are still wanted. * tests.integration: fix 4230c8a * Move the tables of virtual modules to individual documentation pages * Add new doc pages to toctree * Add external ref to windows package manager docs * Improve docstrings * Add documentation on virtual module provider overrides to the module docs * Clarify the scope of the provider param in states. * Add link to provider override docs to all package providers * Add link to provider override docs to all service providers * Add link to provider override docs to all user providers * dd link to provider override docs to all shadow providers * Add link to provider override docs to all group providers * Backport 31164 and 31364 (#32474) * Don't send REQ while another one is waiting for response. The message has to be removed from the queue the only *after* it's already processed to don't confuse send() functionality that expects empty queue means: there's no active sendings. * Fixed zeromq ReqMessageClient destroy * Add link to provider override docs to opkg.py This is a companion to https://github.com/saltstack/salt/pull/32458, but this module was not added until the 2016.3 branch, so the documentation is being updated there for this module. * Add documentation for some master/minion configs (#32454) Refs #32400 Adds docs for: - cli_summary - event_return_queue - event_return_whitelist - event_return_blacklist - file_recv_max_size - fileserver_followsymlinks - fileserver_ignoresymlinks - fileserver_limit_traversal * Automatically detect correct MySQL password column for 5.7 and fix setting passwords (#32440) * Automatically detect MySQL password column * Fix changing password in MySQL 5.7 * Fix lint test * Fix unit tests (?) They will still fail if "authentication_string" is legitimately the right column name, but I don't know what to do about that. * Additional unit test fix * Only unsub if we have a jid Closes #32479 --- src/saltext/consul/pillar/__init__.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a6a8f67..e5d411f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -553,15 +553,24 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: - log.debug( - 'Specified SLS \'%s\' in environment \'%s\' was not ' - 'found. This could be because SLS \'%s\' is in an ' - 'environment other than \'%s\', but \'%s\' is included in ' - 'that environment\'s Pillar top file. It could also be ' - 'due to environment \'%s\' not being defined in ' - '"pillar_roots"', - sls, saltenv, sls, saltenv, saltenv, saltenv - ) + msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' + 'found. '.format(sls, saltenv)) + if self.opts.get('__git_pillar', False) is True: + msg += ( + 'This is likely caused by a git_pillar top file ' + 'containing an environment other than the one for the ' + 'branch in which it resides. Each git_pillar ' + 'branch/tag must have its own top file.' + ) + else: + msg += ( + 'This could be because SLS \'{0}\' is in an ' + 'environment other than \'{1}\', but \'{1}\' is ' + 'included in that environment\'s Pillar top file. It ' + 'could also be due to environment \'{1}\' not being ' + 'defined in \'pillar_roots\'.'.format(sls, saltenv) + ) + log.debug(msg) # return state, mods, errors return None, mods, errors state = None From cb7c8c8b8fb67b2423cf5557bced3be22a25ff23 Mon Sep 17 00:00:00 2001 From: skizunov Date: Wed, 13 Apr 2016 14:55:04 -0400 Subject: [PATCH 437/769] PY3: salt-key --gen-keys fix (#32546) `__init__.py`: - On certain invocations of salt-key (seen when invoked programmatically on Windows without going through the command interpreter `cmd.exe`), `sys.stdin` is None. This has only been seen on Python 3. So `encoding = sys.stdin.encoding` raises an exception in this case. `utils/__init__.py`: - `pem_finger` reads a file in binary mode. So in Python 3, the return type is bytes and not str. Simple fix by using `b''.join` instead of `''.join`. Signed-off-by: Sergey Kizunov --- src/saltext/consul/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 4e4a5c7..faa3975 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -35,7 +35,10 @@ def __define_global_system_encoding_variable__(): # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None - encoding = sys.stdin.encoding + if sys.stdin is not None: + encoding = sys.stdin.encoding + else: + encoding = None if not encoding: # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong From 6419aa1d099ce5a759d80016411401b7b50f817f Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 18 Apr 2016 10:32:44 -0600 Subject: [PATCH 438/769] Lower log level for pillar cache (#32655) This shouldn't show up on salt-call runs --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ae70cb9..578b4df 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -49,7 +49,7 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, 'local': Pillar }.get(opts['file_client'], Pillar) # If local pillar and we're caching, run through the cache system first - log.info('Determining pillar cache') + log.debug('Determining pillar cache') if opts['pillar_cache']: log.info('Compiling pillar from cache') log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) From aabc589b141be06964801542dfe3dc47f35f745d Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Mon, 25 Apr 2016 15:26:09 -0600 Subject: [PATCH 439/769] [2016.3] Merge forward from 2015.8 to 2016.3 (#32784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * json encode arguments passed to an execution module function call this fixes problems where you could pass a string to a module function, which thanks to the yaml decoder which is used when parsing command line arguments could change its type entirely. for example: __salt__['test.echo')('{foo: bar}') the test.echo function just returns the argument it's given. however, because it's being called through a salt-call process like this: salt-call --local test.echo {foo: bar} salt thinks it's yaml and therefore yaml decodes it. the return value from the test.echo call above is therefore a dict, not a string. * Prevent crash if pygit2 package is requesting re-compilation of the e… (#32652) * Prevent crash if pygit2 package is requesting re-compilation of the entire library on production systems (no *devel packages) * Fix PEP8: move imports to the top of the file * Move logger up * Add log error message in case if exception is not an ImportError * align OS grains from older SLES with current one (#32649) * Fixing critical bug to remove only the specified Host instead of the entire Host cluster (#32640) * yumpkg: Ignore epoch in version comparison for explict versions without an epoch (#32563) * yumpkg: Ignore epoch in version comparison for explict versions without an epoch Also properly handle comparisions for packages with multiple versions. Resolves #32229 * Don't attempt downgrade for kernel and its subpackages Multiple versions are supported since their paths do not conflict. * Lower log level for pillar cache (#32655) This shouldn't show up on salt-call runs * Don't access deprecated Exception.message attribute. (#32556) * Don't access deprecated Exception.message attribute. To avoid a deprecation warning message in logs. There is a new function salt.exceptions.get_error_message(e) instead. * Fixed module docs test. * Fix for issue 32523 (#32672) * Fix routes for redhat < 6 * Handle a couple of arguments better (Azure) (#32683) * backporting a fix from develop where the use of splay would result in seconds=0 in the schedule.list when there was no seconds specified in the origina schedule * Handle when beacon not configured and we try to enable/disable them (#32692) * Handle the situation when the beacon is not configured and we try to disable it * a couple more missing returns in the enable & disable * Check dependencies type before appling str operations (#32693) * Update external auth documentation to list supported matcher. (#32733) Thanks to #31598, all matchers are supported for eauth configuration. But we still have no way to use compound matchers in eauth configuration. Update the documentation to explicitly express this limitation. * modules.win_dacl: consistent case of dacl constants (#32720) * Document pillar cache options (#32643) * Add note about Pillar data cache requirement for Pillar targeting method * Add `saltutil.refresh_pillar` function to the scheduled Minion jobs * Minor fixes in docs * Add note about relations between `pillar_cache` option and Pillar Targeting to Master config comments with small reformatting * Document Pillar Cache Options for Salt Master * Document Minions Targeting with Mine * Remove `saltutil.refresh_pillar` scheduled persistent job * Properly handle minion failback failure. (#32749) * Properly handle minion failback failure. Initiate minion restart if all masters down on __master_disconnect like minion does on the initial master connect on start. * Fixed unit test * Improve documentation on pygit2 versions (#32779) This adds an explanation of the python-cffi dep added in pygit2 0.21.0, and recommends 0.20.3 for LTS distros. It also links to the salt-pack issue which tracks the progress of adding pygit2 to our Debian and Ubuntu repositories. * Pylint fix --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fa86bd4..889896d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -49,7 +49,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, env=None, funcs= 'local': Pillar }.get(opts['file_client'], Pillar) # If local pillar and we're caching, run through the cache system first - log.info('Determining pillar cache') + log.debug('Determining pillar cache') if opts['pillar_cache']: log.info('Compiling pillar from cache') log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) From 9e8e08b2db1884c1794995d0faf486e234f7c798 Mon Sep 17 00:00:00 2001 From: steven 'haji' hajducko Date: Thu, 28 Apr 2016 08:57:25 -0700 Subject: [PATCH 440/769] Reload renderers for local pillar after gitfs (#32912) Because of the way the renderers are lazy loaded when using ext_pillar_first and gitfs external pillar, the renderers for the local pillar object were being set to the last loaded gitfs pillar. This was causing the local pillar to fail to render. This small fix forces a reload of the renderers after all the external pillars are done loading. --- src/saltext/consul/pillar/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 889896d..7090191 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -791,6 +791,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): if ext: if self.opts.get('ext_pillar_first', False): self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) + self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge(pillar, From 39c591b7b8f06df94b961772b86b0afb85577304 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 28 Apr 2016 15:49:06 -0600 Subject: [PATCH 441/769] Tests run py3 (#32929) * Zeromq transport python3 * lets toss this fix in here, so that we get exception messages * got states working This forces the serializer to deserialize into strings on py3 but also allows for the argument 'raw' to be passed in allowing for raw bytes to be sent explicitly. I made it this way because the majority of calls are operational and the fileclient is the only thing that needs bytes from the master * all channels need to absorb the new raw arg * Soem redundancy cleanup * fix up base class * try some more... * Fix for "TypeError: send() got an unexpected keyword argument 'raw'" Signed-off-by: Sergey Kizunov * fix an issue where we are making unicode strings bytes for no good reason --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d063e04..61a950e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -733,7 +733,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): errors.append('The "ext_pillar" option is malformed') log.critical(errors[-1]) return {}, errors - if run.keys()[0] in self.opts.get('exclude_ext_pillar', []): + if next(six.iterkeys(run)) in self.opts.get('exclude_ext_pillar', []): continue for key, val in six.iteritems(run): if key not in self.ext_pillars: From dce4d686cb6d7070acec14d7e961f43eff4f115a Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 29 Apr 2016 14:09:25 -0600 Subject: [PATCH 442/769] Add caching loader, with msgpack module (#32953) * Add caching loader, with msgpack module * Update docstrings in loader * Lint --- src/saltext/consul/cache/__init__.py | 129 +++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 src/saltext/consul/cache/__init__.py diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py new file mode 100644 index 0000000..1409eb7 --- /dev/null +++ b/src/saltext/consul/cache/__init__.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +''' +Loader mechanism for caching data, with data expirations, etc. + +.. versionadded:: carbon +''' +from __future__ import absolute_import +import os +import time +from salt.loader import LazyLoader + + +class Cache(object): + ''' + Main caching object + ''' + def __init__(self, opts, driver=None): + self.opts = opts + self.modules = self._modules() + if driver is None: + driver = 'msgpack' + self.driver = driver + + def _modules(self, functions=None, whitelist=None): + ''' + Lazy load the cache modules + ''' + codedir = os.path.dirname(os.path.realpath(__file__)) + return LazyLoader( + [codedir], + self.opts, + tag='cache', + pack={ + '__opts__': self.opts, + '__cache__': functions + }, + whitelist=whitelist, + ) + + def cache(self, bank, key, fun, loop_fun=None, **kwargs): + ''' + Check cache for the data. If it is there, check to see if it needs to + be refreshed. + + If the data is not there, or it needs to be refreshed, then call the + callback function (``fun``) with any given ``**kwargs``. + + In some cases, the callback function returns a list of objects which + need to be processed by a second function. If that is the case, then + the second function is passed in as ``loop_fun``. Each item in the + return list from the first function will be the only argument for the + second function. + ''' + expire_seconds = kwargs.get('expire', 86400) # 1 day + + updated = self.updated(bank, key) + update_cache = False + if updated is None: + update_cache = True + else: + if int(time.time()) - updated > expire_seconds: + update_cache = True + + data = self.fetch(bank, key) + + if data is None or update_cache is True: + if loop_fun is not None: + data = [] + items = fun(**kwargs) + for item in items: + data.append(loop_fun(item)) + else: + data = fun(**kwargs) + self.store(bank, key, data) + + return data + + def store(self, bank, key, data): + ''' + Store data using the specified module + + bank + The name of the location inside the cache which will hold the key + and its associated data. + + Bank names should be formatted in a way that can be used as a + directory structure. If slashes are included in the name, then they + refer to a nested directory structure (meaning, directories will be + created to accomodate the name). + + key + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + + data + The data which will be stored in the cache. This data should be + in a format which can be serialized by msgpack/json/yaml/etc. + ''' + fun = '{0}.{1}'.format(self.driver, 'store') + return self.modules[fun](bank, key, data) + + def fetch(self, bank, key): + ''' + Fetch data using the specified module + + bank + The name of the location inside the cache which will hold the key + and its associated data. + + Bank names should be formatted in a way that can be used as a + directory structure. If slashes are included in the name, then they + refer to a nested directory structure (meaning, directories will be + created to accomodate the name). + + key + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + ''' + fun = '{0}.{1}'.format(self.driver, 'fetch') + return self.modules[fun](bank, key) + + def updated(self, bank, key): + ''' + Return the last updated epoch for the specified key + ''' + fun = '{0}.{1}'.format(self.driver, 'updated') + return self.modules[fun](bank, key) From b5c1fe9ebd256485612466c1e4ccac1ac55fcb7b Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Thu, 12 May 2016 14:56:07 -0600 Subject: [PATCH 443/769] Fix list_networks() when a resource_group has no networks (#33208) --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 1409eb7..8afed05 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -63,7 +63,7 @@ def cache(self, bank, key, fun, loop_fun=None, **kwargs): data = self.fetch(bank, key) - if data is None or update_cache is True: + if not data or update_cache is True: if loop_fun is not None: data = [] items = fun(**kwargs) From 7d957ed9e231d83bba751c07f1e769daf8903899 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 20 May 2016 20:41:51 +0300 Subject: [PATCH 444/769] Allow to blacklist/whitelist renderers. (#33358) * Allow to blacklist/whitelist renderers. * Fix tests. * Unit test for renderer blacklisting. * Minor tests fixes --- src/saltext/consul/pillar/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4334dd5..c5a504f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -370,8 +370,10 @@ def get_tops(self): ), self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], self.opts['pillarenv'], - _pillar_rend=True + _pillar_rend=True, ) ] else: @@ -386,8 +388,10 @@ def get_tops(self): top, self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv=saltenv, - _pillar_rend=True + _pillar_rend=True, ) ) except Exception as exc: @@ -424,8 +428,10 @@ def get_tops(self): ).get('dest', False), self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv=saltenv, - _pillar_rend=True + _pillar_rend=True, ) ) except Exception as exc: @@ -578,6 +584,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state = compile_template(fn_, self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv, sls, _pillar_rend=True, From 41cda7f3f5e932d16085ccd9112f752cee3cfaa1 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 27 May 2016 10:05:13 -0700 Subject: [PATCH 445/769] Merge remote-tracking branch 'saltstack/develop' into develop # Conflicts: # salt/cloud/__init__.py --- .../pillar/all/salt.pillar.consul_pillar.rst | 7 +- docs/ref/sdb/all/salt.sdb.consul.rst | 5 + src/saltext/consul/__init__.py | 7 +- src/saltext/consul/cache/__init__.py | 129 ++++++ src/saltext/consul/modules/consul.py | 59 ++- src/saltext/consul/pillar/__init__.py | 385 ++++++++++++------ src/saltext/consul/pillar/consul_pillar.py | 142 ++++++- src/saltext/consul/sdb/consul.py | 80 ++++ 8 files changed, 647 insertions(+), 167 deletions(-) create mode 100644 docs/ref/sdb/all/salt.sdb.consul.rst create mode 100644 src/saltext/consul/cache/__init__.py create mode 100644 src/saltext/consul/sdb/consul.py diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst index 523ff0f..c27449b 100644 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -1,6 +1,5 @@ -========================= -salt.pillar.consul_pillar -========================= +salt.pillar.consul_pillar module +================================ .. automodule:: salt.pillar.consul_pillar - :members: \ No newline at end of file + :members: diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst new file mode 100644 index 0000000..5a47d2b --- /dev/null +++ b/docs/ref/sdb/all/salt.sdb.consul.rst @@ -0,0 +1,5 @@ +salt.sdb.consul module +====================== + +.. automodule:: salt.sdb.consul + :members: diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9f571fb..faa3975 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -35,7 +35,10 @@ def __define_global_system_encoding_variable__(): # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None - encoding = sys.stdin.encoding + if sys.stdin is not None: + encoding = sys.stdin.encoding + else: + encoding = None if not encoding: # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong @@ -60,7 +63,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins # pylint: disable=incompatible-py3-code + import __builtin__ as builtins else: import builtins # pylint: disable=import-error diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py new file mode 100644 index 0000000..8afed05 --- /dev/null +++ b/src/saltext/consul/cache/__init__.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +''' +Loader mechanism for caching data, with data expirations, etc. + +.. versionadded:: carbon +''' +from __future__ import absolute_import +import os +import time +from salt.loader import LazyLoader + + +class Cache(object): + ''' + Main caching object + ''' + def __init__(self, opts, driver=None): + self.opts = opts + self.modules = self._modules() + if driver is None: + driver = 'msgpack' + self.driver = driver + + def _modules(self, functions=None, whitelist=None): + ''' + Lazy load the cache modules + ''' + codedir = os.path.dirname(os.path.realpath(__file__)) + return LazyLoader( + [codedir], + self.opts, + tag='cache', + pack={ + '__opts__': self.opts, + '__cache__': functions + }, + whitelist=whitelist, + ) + + def cache(self, bank, key, fun, loop_fun=None, **kwargs): + ''' + Check cache for the data. If it is there, check to see if it needs to + be refreshed. + + If the data is not there, or it needs to be refreshed, then call the + callback function (``fun``) with any given ``**kwargs``. + + In some cases, the callback function returns a list of objects which + need to be processed by a second function. If that is the case, then + the second function is passed in as ``loop_fun``. Each item in the + return list from the first function will be the only argument for the + second function. + ''' + expire_seconds = kwargs.get('expire', 86400) # 1 day + + updated = self.updated(bank, key) + update_cache = False + if updated is None: + update_cache = True + else: + if int(time.time()) - updated > expire_seconds: + update_cache = True + + data = self.fetch(bank, key) + + if not data or update_cache is True: + if loop_fun is not None: + data = [] + items = fun(**kwargs) + for item in items: + data.append(loop_fun(item)) + else: + data = fun(**kwargs) + self.store(bank, key, data) + + return data + + def store(self, bank, key, data): + ''' + Store data using the specified module + + bank + The name of the location inside the cache which will hold the key + and its associated data. + + Bank names should be formatted in a way that can be used as a + directory structure. If slashes are included in the name, then they + refer to a nested directory structure (meaning, directories will be + created to accomodate the name). + + key + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + + data + The data which will be stored in the cache. This data should be + in a format which can be serialized by msgpack/json/yaml/etc. + ''' + fun = '{0}.{1}'.format(self.driver, 'store') + return self.modules[fun](bank, key, data) + + def fetch(self, bank, key): + ''' + Fetch data using the specified module + + bank + The name of the location inside the cache which will hold the key + and its associated data. + + Bank names should be formatted in a way that can be used as a + directory structure. If slashes are included in the name, then they + refer to a nested directory structure (meaning, directories will be + created to accomodate the name). + + key + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + ''' + fun = '{0}.{1}'.format(self.driver, 'fetch') + return self.modules[fun](bank, key) + + def updated(self, bank, key): + ''' + Return the last updated epoch for the specified key + ''' + fun = '{0}.{1}'.format(self.driver, 'updated') + return self.modules[fun](bank, key) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 80ec9e2..4bbf229 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -26,6 +26,11 @@ from salt.exceptions import SaltInvocationError +# Don't shadow built-ins. +__func_alias__ = { + 'list_': 'list' +} + __virtualname__ = 'consul' @@ -63,6 +68,10 @@ def _query(function, base_url = _urljoin(consul_url, '{0}/'.format(api_version)) url = _urljoin(base_url, function, False) + if data is None: + data = {} + data = json.dumps(data) + result = salt.utils.http.query( url, method=method, @@ -92,7 +101,7 @@ def _query(function, return ret -def list(consul_url=None, key=None, **kwargs): +def list_(consul_url=None, key=None, **kwargs): ''' List keys in Consul @@ -160,13 +169,15 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): salt '*' consul.list key='web' recurse='True' decode='True' - By default values stored in Consul are base64 encoded, passing the - decode option will show them as the decoded values. + By default values stored in Consul are base64 encoded, passing the + decode option will show them as the decoded values. + + .. code-block:: bash salt '*' consul.list key='web' recurse='True' decode='True' raw='True' - By default Consult will return other information about the key, the raw - option will return only the raw value. + By default Consult will return other information about the key, the raw + option will return only the raw value. ''' ret = {} @@ -302,7 +313,7 @@ def put(consul_url=None, key=None, value=None, **kwargs): ret = _query(consul_url=consul_url, function=function, method=method, - data=json.dumps(data), + data=data, query_params=query_params) if ret['res']: @@ -688,9 +699,7 @@ def agent_check_register(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] else: - ret['message'] = 'Required parameter "name" is missing.' - ret['res'] = False - return ret + raise SaltInvocationError('Required argument "name" is missing.') if True not in [True for item in ('script', 'http') if item in kwargs]: ret['message'] = 'Required parameter "script" or "http" is missing.' @@ -966,6 +975,8 @@ def agent_service_register(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'address' in kwargs: data['Address'] = kwargs['address'] @@ -1024,7 +1035,7 @@ def agent_service_deregister(consul_url=None, serviceid=None): Used to remove a service. :param consul_url: The Consul server URL. - :param name: A name describing the service. + :param serviceid: A serviceid describing the service. :return: Boolean and message indicating success or failure. CLI Example: @@ -1169,6 +1180,8 @@ def session_create(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'checks' in kwargs: data['Touch'] = kwargs['touch'] @@ -1186,7 +1199,7 @@ def session_create(consul_url=None, **kwargs): if str(_ttl).endswith('s'): _ttl = _ttl[:-1] - if not int(_ttl) >= 0 and not int(_ttl) <= 3600: + if int(_ttl) < 0 or int(_ttl) > 3600: ret['message'] = ('TTL must be ', 'between 0 and 3600.') ret['res'] = False @@ -1448,11 +1461,11 @@ def catalog_register(consul_url=None, **kwargs): if res['res']: ret['res'] = True ret['message'] = ('Catalog registration ' - 'for {0} successful.'.format(kwargs['name'])) + 'for {0} successful.'.format(kwargs['node'])) else: ret['res'] = False ret['message'] = ('Catalog registration ' - 'for {0} failed.'.format(kwargs['name'])) + 'for {0} failed.'.format(kwargs['node'])) return ret @@ -1509,11 +1522,11 @@ def catalog_deregister(consul_url=None, **kwargs): data=data) if res['res']: ret['res'] = True - ret['message'] = 'Catalog item {0} removed.'.format(kwargs['name']) + ret['message'] = 'Catalog item {0} removed.'.format(kwargs['node']) else: ret['res'] = False ret['message'] = ('Removing Catalog ' - 'item {0} failed.'.format(kwargs['name'])) + 'item {0} failed.'.format(kwargs['node'])) return ret @@ -1973,6 +1986,8 @@ def acl_create(consul_url=None, **kwargs): if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2030,12 +2045,14 @@ def acl_update(consul_url=None, **kwargs): if 'id' in kwargs: data['ID'] = kwargs['id'] else: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret if 'name' in kwargs: data['Name'] = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2086,7 +2103,7 @@ def acl_delete(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2133,7 +2150,7 @@ def acl_info(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2172,7 +2189,7 @@ def acl_clone(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2217,7 +2234,7 @@ def acl_list(consul_url=None, **kwargs): return ret if 'id' not in kwargs: - ret['message'] = 'Required paramter "id" is missing.' + ret['message'] = 'Required parameter "id" is missing.' ret['res'] = False return ret @@ -2318,6 +2335,8 @@ def event_list(consul_url=None, **kwargs): if 'name' in kwargs: query_params = kwargs['name'] + else: + raise SaltInvocationError('Required argument "name" is missing.') function = 'event/list/' ret = _query(consul_url=consul_url, diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index cbc4076..c5a504f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -9,6 +9,7 @@ import os import collections import logging +import tornado.gen # Import salt libs import salt.loader @@ -17,6 +18,7 @@ import salt.crypt import salt.transport import salt.utils.url +import salt.utils.cache from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -26,51 +28,40 @@ # Import 3rd-party libs import salt.ext.six as six -import tornado.gen - log = logging.getLogger(__name__) -def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, +def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' - if env is not None: - salt.utils.warn_until( - 'Boron', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Boron.' - ) - # Backwards compatibility - saltenv = env ptype = { 'remote': RemotePillar, 'local': Pillar }.get(opts['file_client'], Pillar) - return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + # If local pillar and we're caching, run through the cache system first + log.debug('Determining pillar cache') + if opts['pillar_cache']: + log.info('Compiling pillar from cache') + log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) + return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, + pillar=pillar, pillarenv=pillarenv) + return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) # TODO: migrate everyone to this one! -def get_async_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None, +def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' - if env is not None: - salt.utils.warn_until( - 'Boron', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt Boron.' - ) - # Backwards compatibility - saltenv = env ptype = { 'remote': AsyncRemotePillar, 'local': AsyncPillar, }.get(opts['file_client'], AsyncPillar) - return ptype(opts, grains, id_, saltenv, ext, functions=funcs, + return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) @@ -78,15 +69,16 @@ class AsyncRemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains - self.id_ = id_ + self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) - self.opts['pillarenv'] = pillarenv + if pillarenv is not None or 'pillarenv' not in self.opts: + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -99,7 +91,7 @@ def compile_pillar(self): ''' Return a future which will contain the pillar data from the master ''' - load = {'id': self.id_, + load = {'id': self.minion_id, 'grains': self.grains, 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], @@ -108,10 +100,14 @@ def compile_pillar(self): 'cmd': '_pillar'} if self.ext: load['ext'] = self.ext - ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( - load, - dictkey='pillar', - ) + try: + ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( + load, + dictkey='pillar', + ) + except: + log.exception('Exception getting pillar:') + raise SaltClientError('Exception getting pillar.') if not isinstance(ret_pillar, dict): msg = ('Got a bad pillar from master, type {0}, expecting dict: ' @@ -126,15 +122,16 @@ class RemotePillar(object): ''' Get the pillar from the master ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext self.grains = grains - self.id_ = id_ + self.minion_id = minion_id self.channel = salt.transport.Channel.factory(opts) - self.opts['pillarenv'] = pillarenv + if pillarenv is not None or 'pillarenv' not in self.opts: + self.opts['pillarenv'] = pillarenv self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -146,7 +143,7 @@ def compile_pillar(self): ''' Return the pillar data from the master ''' - load = {'id': self.id_, + load = {'id': self.minion_id, 'grains': self.grains, 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], @@ -168,16 +165,106 @@ def compile_pillar(self): return ret_pillar +class PillarCache(object): + ''' + Return a cached pillar if it exists, otherwise cache it. + + Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs. + Each saltenv contains a pillar dict + + Example data structure: + + ``` + {'minion_1': + {'base': {'pilar_key_1' 'pillar_val_1'} + } + ''' + # TODO ABC? + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, + pillar=None, pillarenv=None): + # Yes, we need all of these because we need to route to the Pillar object + # if we have no cache. This is another refactor target. + + # Go ahead and assign these because they may be needed later + self.opts = opts + self.grains = grains + self.minion_id = minion_id + self.ext = ext + self.functions = functions + self.pillar = pillar + self.pillarenv = pillarenv + + if saltenv is None: + self.saltenv = 'base' + else: + self.saltenv = saltenv + + # Determine caching backend + self.cache = salt.utils.cache.CacheFactory.factory( + self.opts['pillar_cache_backend'], + self.opts['pillar_cache_ttl'], + minion_cache_path=self._minion_cache_path(minion_id)) + + def _minion_cache_path(self, minion_id): + ''' + Return the path to the cache file for the minion. + + Used only for disk-based backends + ''' + return os.path.join(self.opts['cachedir'], 'pillar_cache', minion_id) + + def fetch_pillar(self): + ''' + In the event of a cache miss, we need to incur the overhead of caching + a new pillar. + ''' + log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) + fresh_pillar = Pillar(self.opts, + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar=self.pillar, + pillarenv=self.pillarenv) + return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here + + def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs + log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + log.debug('Scanning cache: {0}'.format(self.cache._dict)) + # Check the cache! + if self.minion_id in self.cache: # Keyed by minion_id + # TODO Compare grains, etc? + if self.saltenv in self.cache[self.minion_id]: + # We have a cache hit! Send it back. + log.debug('Pillar cache hit for minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + return self.cache[self.minion_id][self.saltenv] + else: + # We found the minion but not the env. Store it. + fresh_pillar = self.fetch_pillar() + self.cache[self.minion_id][self.saltenv] = fresh_pillar + log.debug('Pillar cache miss for saltenv {0} for minion {1}'.format(self.saltenv, self.minion_id)) + return fresh_pillar + else: + # We haven't seen this minion yet in the cache. Store it. + fresh_pillar = self.fetch_pillar() + self.cache[self.minion_id] = {self.saltenv: fresh_pillar} + log.debug('Pillar cache miss for minion {0}'.format(self.minion_id)) + log.debug('Current pillar cache: {0}'.format(self.cache._dict)) # FIXME hack! + return fresh_pillar + + class Pillar(object): ''' Read over the pillar top files and render the pillar data ''' - def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, + def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): + self.minion_id = minion_id # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, id_, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -195,10 +282,13 @@ def __init__(self, opts, grains, id_, saltenv, ext=None, functions=None, self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) + ext_pillar_opts = copy.deepcopy(self.opts) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 - ext_pillar_opts = dict(self.opts) ext_pillar_opts['file_roots'] = self.actual_file_roots + # Keep the incoming opts ID intact, ie, the master id + if 'id' in opts: + ext_pillar_opts['id'] = opts['id'] self.merge_strategy = 'smart' if opts.get('pillar_source_merging_strategy'): self.merge_strategy = opts['pillar_source_merging_strategy'] @@ -223,29 +313,20 @@ def __valid_ext(self, ext): return {} return ext - def __gen_opts(self, opts_in, grains, id_, saltenv=None, ext=None, env=None, pillarenv=None): + def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' - if env is not None: - salt.utils.warn_until( - 'Boron', - 'Passing a salt environment should be done using \'saltenv\' ' - 'not \'env\'. This functionality will be removed in Salt ' - 'Boron.' - ) - # Backwards compatibility - saltenv = env - opts = dict(opts_in) + opts = copy.deepcopy(opts_in) opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' if not grains: opts['grains'] = {} else: opts['grains'] = grains - opts['id'] = id_ if 'environment' not in opts: opts['environment'] = saltenv + opts['id'] = self.minion_id if 'pillarenv' not in opts: opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): @@ -289,8 +370,10 @@ def get_tops(self): ), self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], self.opts['pillarenv'], - _pillar_rend=True + _pillar_rend=True, ) ] else: @@ -305,15 +388,17 @@ def get_tops(self): top, self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv=saltenv, - _pillar_rend=True + _pillar_rend=True, ) ) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) - log.error('Pillar rendering failed for minion {0}: '.format(self.opts['id']), + log.error('Pillar rendering failed for minion {0}: '.format(self.minion_id), exc_info=True) # Search initial top files for includes @@ -343,8 +428,10 @@ def get_tops(self): ).get('dest', False), self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv=saltenv, - _pillar_rend=True + _pillar_rend=True, ) ) except Exception as exc: @@ -391,7 +478,9 @@ def merge_tops(self, tops): if isinstance(comp, six.string_types): states[comp] = True if ignore_missing: - self.ignored_pillars[saltenv] = list(states.keys()) + if saltenv not in self.ignored_pillars: + self.ignored_pillars[saltenv] = [] + self.ignored_pillars[saltenv].extend(states.keys()) top[saltenv][tgt] = matches top[saltenv][tgt].extend(states) return self.sort_top_targets(top, orders) @@ -442,10 +531,12 @@ def top_matches(self, top): self.opts.get('nodegroups', {}), ): if saltenv not in matches: - matches[saltenv] = [] + matches[saltenv] = env_matches = [] + else: + env_matches = matches[saltenv] for item in data: - if isinstance(item, six.string_types): - matches[saltenv].append(item) + if isinstance(item, six.string_types) and item not in env_matches: + env_matches.append(item) return matches def render_pstate(self, sls, saltenv, mods, defaults=None): @@ -468,10 +559,24 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: - log.debug('Specified SLS \'{0}\' in environment \'{1}\' is not' - ' found, which might be due to environment \'{1}\'' - ' not being present in "pillar_roots" yet!' - .format(sls, saltenv)) + msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' + 'found. '.format(sls, saltenv)) + if self.opts.get('__git_pillar', False) is True: + msg += ( + 'This is likely caused by a git_pillar top file ' + 'containing an environment other than the one for the ' + 'branch in which it resides. Each git_pillar ' + 'branch/tag must have its own top file.' + ) + else: + msg += ( + 'This could be because SLS \'{0}\' is in an ' + 'environment other than \'{1}\', but \'{1}\' is ' + 'included in that environment\'s Pillar top file. It ' + 'could also be due to environment \'{1}\' not being ' + 'defined in \'pillar_roots\'.'.format(sls, saltenv) + ) + log.debug(msg) # return state, mods, errors return None, mods, errors state = None @@ -479,6 +584,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state = compile_template(fn_, self.rend, self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], saltenv, sls, _pillar_rend=True, @@ -524,29 +631,31 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): mods, defaults ) - if nstate: - if key: - nstate = { - key: nstate - } - - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get('renderer', 'yaml')) - - if err: - errors += err + if nstate: + if key: + nstate = { + key: nstate + } + + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + + if err: + errors += err return state, mods, errors - def render_pillar(self, matches): + def render_pillar(self, matches, errors=None): ''' Extract the sls pillar files from the matches and render them into the pillar ''' pillar = copy.copy(self.pillar_override) - errors = [] + if errors is None: + errors = [] for saltenv, pstates in six.iteritems(matches): mods = set() for sls in pstates: @@ -573,7 +682,8 @@ def render_pillar(self, matches): pillar, pstate, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) return pillar, errors @@ -583,85 +693,80 @@ def _external_pillar_data(self, pillar, val, pillar_dirs, key): ''' ext = None - # try the new interface, which includes the minion ID - # as first argument if isinstance(val, dict): - ext = self.ext_pillars[key](self.opts['id'], pillar, **val) + ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): if key == 'git': - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, val, pillar_dirs) else: - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, pillar, *val) else: if key == 'git': - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, val, pillar_dirs) else: - ext = self.ext_pillars[key](self.opts['id'], + ext = self.ext_pillars[key](self.minion_id, pillar, val) return ext - def ext_pillar(self, pillar, pillar_dirs): + def ext_pillar(self, pillar, pillar_dirs, errors=None): ''' Render the external pillar data ''' + if errors is None: + errors = [] if 'ext_pillar' not in self.opts: - return pillar + return pillar, errors if not isinstance(self.opts['ext_pillar'], list): - log.critical('The "ext_pillar" option is malformed') - return pillar + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return pillar, errors ext = None # Bring in CLI pillar data - pillar.update(self.pillar_override) + if self.pillar_override and isinstance(self.pillar_override, dict): + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + for run in self.opts['ext_pillar']: if not isinstance(run, dict): - log.critical('The "ext_pillar" option is malformed') - return {} + errors.append('The "ext_pillar" option is malformed') + log.critical(errors[-1]) + return {}, errors + if next(six.iterkeys(run)) in self.opts.get('exclude_ext_pillar', []): + continue for key, val in six.iteritems(run): if key not in self.ext_pillars: - err = ('Specified ext_pillar interface {0} is ' - 'unavailable').format(key) - log.critical(err) + log.critical( + 'Specified ext_pillar interface {0} is ' + 'unavailable'.format(key) + ) continue try: - try: - ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) - except TypeError as exc: - if str(exc).startswith('ext_pillar() takes exactly '): - log.warning('Deprecation warning: ext_pillar "{0}"' - ' needs to accept minion_id as first' - ' argument'.format(key)) - else: - raise - - ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) + ext = self._external_pillar_data(pillar, + val, + pillar_dirs, + key) except Exception as exc: - log.exception( - 'Failed to load ext_pillar {0}: {1}'.format( - key, - exc - ) - ) + errors.append('Failed to load ext_pillar {0}: {1}'.format( + key, exc)) if ext: pillar = merge( pillar, ext, self.merge_strategy, - self.opts.get('renderer', 'yaml')) + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) ext = None - return pillar + return pillar, errors def compile_pillar(self, ext=True, pillar_dirs=None): ''' @@ -669,23 +774,37 @@ def compile_pillar(self, ext=True, pillar_dirs=None): ''' top, top_errors = self.get_top() if ext: - if self.opts.get('ext_pillar_first', False): - self.opts['pillar'] = self.ext_pillar({}, pillar_dirs) + if self.opts.get('pillar_roots_override_ext_pillar', False) or self.opts.get('ext_pillar_first', False): + salt.utils.warn_until('Nitrogen', + 'The \'ext_pillar_first\' option has been deprecated and ' + 'replaced by \'pillar_roots_override_ext_pillar\'.' + ) + self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) + self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) - pillar, errors = self.render_pillar(matches) - pillar = merge(pillar, - self.opts['pillar'], - self.merge_strategy, - self.opts.get('renderer', 'yaml')) + pillar, errors = self.render_pillar(matches, errors=errors) + if self.opts.get('pillar_roots_override_ext_pillar', False): + pillar = merge(self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + else: + pillar = merge(pillar, + self.opts['pillar'], + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar = self.ext_pillar(pillar, pillar_dirs) + pillar, errors = self.ext_pillar( + pillar, pillar_dirs, errors=errors) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) errors.extend(top_errors) - if self.opts.get('pillar_opts', True): + if self.opts.get('pillar_opts', False): mopts = dict(self.opts) if 'grains' in mopts: mopts.pop('grains') @@ -697,6 +816,14 @@ def compile_pillar(self, ext=True, pillar_dirs=None): for error in errors: log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors + + if self.pillar_override and isinstance(self.pillar_override, dict): + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + return pillar diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index df9b618..3a421c2 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Use consul data as a Pillar source +Use Consul K/V as a Pillar source with values parsed as YAML :depends: - python-consul @@ -12,6 +12,54 @@ my_consul_config: consul.host: 127.0.0.1 consul.port: 8500 + consul.token: b6376760-a8bb-edd5-fcda-33bc13bfc556 + consul.scheme: http + consul.consistency: default + consul.dc: dev + consul.verify: True + +All parameters are optional. + +If you have a multi-datacenter Consul cluster you can map your ``pillarenv``s +to your data centers by providing a dictionary of mappings in ``consul.dc`` +field: + +.. code-block:: yaml + + my_consul_config: + consul.dc: + dev: us-east-1 + prod: us-west-1 + +In the example above we specifying static mapping between Pillar environments +and data centers: the data for ``dev`` and ``prod`` Pillar environments will +be fetched from ``us-east-1`` and ``us-west-1`` datacenter respectively. + +In fact when ``consul.dc`` is set to dictionary keys are processed as regular +expressions (that can capture named parameters) and values are processed as +string templates as per PEP 3101. + +.. code-block:: yaml + + my_consul_config: + consul.dc: + ^dev-.*$: dev-datacenter + ^(?P.*)-prod$: prod-datacenter-{region} + +This example maps all Pillar environments starting with ``dev-`` to +``dev-datacenter`` whereas Pillar environment like ``eu-prod`` will be +mapped to ``prod-datacenter-eu``. + +Before evaluation patterns are sorted by length in descending order. + +If Pillar environment names correspond to data center names a single pattern +can be used: + +.. code-block:: yaml + + my_consul_config: + consul.dc: + ^(?P.*)$: '{env}' After the profile is created, configure the external pillar system to use it. Optionally, a root may be specified. @@ -32,13 +80,14 @@ - consul: my_consul_config - consul: my_other_consul_config -The ``minion_id`` may be used in the ``root`` path to expose minion-specific -information stored in consul. +Either the ``minion_id``, or the ``role`` grain may be used in the ``root`` +path to expose minion-specific information stored in consul. .. code-block:: yaml ext_pillar: - consul: my_consul_config root=/salt/%(minion_id)s + - consul: my_consul_config root=/salt/%(role)s Minion-specific values may override shared values when the minion-specific root appears after the shared root: @@ -49,6 +98,13 @@ - consul: my_consul_config root=/salt-shared - consul: my_other_consul_config root=/salt-private/%(minion_id)s +If using the ``role`` grain in the consul key path, be sure to define it using +`/etc/salt/grains`, or similar: + +.. code-block:: yaml + + role: my-minion-role + ''' from __future__ import absolute_import @@ -57,6 +113,8 @@ import re +import yaml + from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge @@ -97,9 +155,11 @@ def ext_pillar(minion_id, if len(comps) > 1 and comps[1].startswith('root='): path = comps[1].replace('root=', '') + role = __salt__['grains.get']('role') # put the minion's ID in the path if necessary path %= { - 'minion_id': minion_id + 'minion_id': minion_id, + 'role': role } try: @@ -131,6 +191,8 @@ def fetch_tree(client, path): log.debug('Fetched items: %r', format(items)) + if items is None: + return ret for item in reversed(items): key = re.sub(r'^' + path + '/?', '', item['Key']) if key != "": @@ -148,13 +210,15 @@ def pillar_format(ret, keys, value): Perform data formatting to be used as pillar data and merge it with the current pillar data ''' + # if value is empty in Consul then it's None here - skip it if value is None: return ret - if value[0] == value[-1] == '"': - pillar_value = value[1:-1] - else: - array_data = value.split('\n') - pillar_value = array_data[0] if len(array_data) == 1 else array_data + + # If value is not None then it's a string + # Use YAML to parse the data + # YAML strips whitespaces unless they're surrounded by quotes + pillar_value = yaml.load(value) + keyvalue = keys.pop() pil = {keyvalue: pillar_value} keys.reverse() @@ -182,14 +246,68 @@ def get_conn(opts, profile): else: conf = opts_merged - consul_host = conf.get('consul.host', '127.0.0.1') - consul_port = conf.get('consul.port', 8500) + params = {} + for key in ('host', 'port', 'token', 'scheme', 'consistency', 'dc', 'verify'): + prefixed_key = 'consul.{key}'.format(key=key) + if prefixed_key in conf: + params[key] = conf[prefixed_key] + + if 'dc' in params: + pillarenv = opts_merged.get('pillarenv') or 'base' + params['dc'] = _resolve_datacenter(params['dc'], pillarenv) if HAS_CONSUL: - return consul.Consul(host=consul_host, port=consul_port) + return consul.Consul(**params) else: raise CommandExecutionError( '(unable to import consul, ' 'module most likely not installed. Download python-consul ' 'module and be sure to import consul)' ) + + +def _resolve_datacenter(dc, pillarenv): + ''' + If ``dc`` is a string - return it as is. + + If it's a dict then sort it in descending order by key length and try + to use keys as RegEx patterns to match against ``pillarenv``. + The value for matched pattern should be a string (that can use + ``str.format`` syntax togetehr with captured variables from pattern) + pointing to targe data center to use. + + If none patterns matched return ``None`` which meanse us datacenter of + conencted Consul agent. + ''' + log.debug("Resolving Consul datacenter based on: {dc}".format(dc=dc)) + + try: + mappings = dc.items() # is it a dict? + except AttributeError: + log.debug('Using pre-defined DC: {dc!r}'.format(dc=dc)) + return dc + + log.debug( + 'Selecting DC based on pillarenv using {num} pattern(s)'.format( + num=len(mappings) + ) + ) + log.debug('Pillarenv set to {env!r}'.format(env=pillarenv)) + + # sort in reverse based on pattern length + # but use alphabetic order within groups of patterns of same length + sorted_mappings = sorted(mappings, key=lambda m: (-len(m[0]), m[0])) + + for pattern, target in sorted_mappings: + match = re.match(pattern, pillarenv) + if match: + log.debug('Matched pattern: {pattern!r}'.format(pattern=pattern)) + result = target.format(**match.groupdict()) + log.debug('Resolved datacenter: {result!r}'.format(result=result)) + return result + + log.debug( + 'None of following patterns matched pillarenv={env}: {lst}'.format( + env=pillarenv, lst=', '.join(repr(x) for x in mappings) + ) + ) diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py new file mode 100644 index 0000000..4750164 --- /dev/null +++ b/src/saltext/consul/sdb/consul.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +''' +Consul sdb Module + +:maintainer: SaltStack +:maturity: New +:platform: all + +This module allows access to Consul using an ``sdb://`` URI + +Like all sdb modules, the Consul module requires a configuration profile to +be configured in either the minion or master configuration file. This profile +requires very little. For example: + +.. code-block:: yaml + myconsul: + driver: consul + host: 127.0.0.1 + port: 8500 + token: b6376760-a8bb-edd5-fcda-33bc13bfc556 + scheme: http + consistency: default + dc: dev + verify: True + +The ``driver`` refers to the Consul module, all other options are optional. +For option details see: http://python-consul.readthedocs.org/en/latest/#consul +''' +from __future__ import absolute_import + +from salt.exceptions import CommandExecutionError + +try: + import consul + HAS_CONSUL = True +except ImportError: + HAS_CONSUL = False + + +__func_alias__ = { + 'set_': 'set' +} + + +def set_(key, value, profile=None): + if not profile: + return False + + conn = get_conn(profile) + + return conn.kv.put(key, value) + + +def get(key, profile=None): + if not profile: + return False + + conn = get_conn(profile) + + _, result = conn.kv.get(key) + + return result['Value'] if result else None + + +def get_conn(profile): + ''' + Return a client object for accessing consul + ''' + params = {} + for key in ('host', 'port', 'token', 'scheme', 'consistency', 'dc', 'verify'): + if key in profile: + params[key] = profile[key] + + if HAS_CONSUL: + return consul.Consul(**params) + else: + raise CommandExecutionError( + '(unable to import consul, ' + 'module most likely not installed. PLease install python-consul)' + ) From 40e487688b84362df4fb645099ca95a5d68c5fb9 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Sat, 11 Jun 2016 12:33:37 +0100 Subject: [PATCH 446/769] Convert readthedocs links for their .org -> .io migration for hosted projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per [their blog post of the 27th April](https://blog.readthedocs.com/securing-subdomains/) ‘Securing subdomains’: > Starting today, Read the Docs will start hosting projects from subdomains on the domain readthedocs.io, instead of on readthedocs.org. This change addresses some security concerns around site cookies while hosting user generated data on the same domain as our dashboard. Test Plan: Manually visited all the links I’ve modified. --- src/saltext/consul/sdb/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py index 4750164..16cce0e 100644 --- a/src/saltext/consul/sdb/consul.py +++ b/src/saltext/consul/sdb/consul.py @@ -24,7 +24,7 @@ verify: True The ``driver`` refers to the Consul module, all other options are optional. -For option details see: http://python-consul.readthedocs.org/en/latest/#consul +For option details see: https://python-consul.readthedocs.io/en/latest/#consul ''' from __future__ import absolute_import From c8200c02f986280eeefc01c140ec87567f4a7809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Mon, 13 Jun 2016 17:24:39 +0200 Subject: [PATCH 447/769] Dynamically support params in Consul function call --- src/saltext/consul/pillar/consul_pillar.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 3a421c2..fd97a68 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -247,10 +247,9 @@ def get_conn(opts, profile): conf = opts_merged params = {} - for key in ('host', 'port', 'token', 'scheme', 'consistency', 'dc', 'verify'): - prefixed_key = 'consul.{key}'.format(key=key) - if prefixed_key in conf: - params[key] = conf[prefixed_key] + for key in conf: + if key.startswith('consul.'): + params[key.split('.')[1]] = conf[key] if 'dc' in params: pillarenv = opts_merged.get('pillarenv') or 'base' From 06f3dae4bdf2cfbf580685c0401398f94e697a62 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 15 Jun 2016 10:41:24 -0600 Subject: [PATCH 448/769] expose environment specific information stored in consul (#34027) added support so that the environment grain may be used in the root path to expose environment-specific information stored in consul. --- src/saltext/consul/pillar/consul_pillar.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index fd97a68..bd367de 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -80,7 +80,7 @@ - consul: my_consul_config - consul: my_other_consul_config -Either the ``minion_id``, or the ``role`` grain may be used in the ``root`` +Either the ``minion_id``, or the ``role``, or the ``environment`` grain may be used in the ``root`` path to expose minion-specific information stored in consul. .. code-block:: yaml @@ -88,6 +88,7 @@ ext_pillar: - consul: my_consul_config root=/salt/%(minion_id)s - consul: my_consul_config root=/salt/%(role)s + - consul: my_consul_config root=/salt/%(environment)s Minion-specific values may override shared values when the minion-specific root appears after the shared root: @@ -98,12 +99,13 @@ - consul: my_consul_config root=/salt-shared - consul: my_other_consul_config root=/salt-private/%(minion_id)s -If using the ``role`` grain in the consul key path, be sure to define it using +If using the ``role`` or ``environment`` grain in the consul key path, be sure to define it using `/etc/salt/grains`, or similar: .. code-block:: yaml role: my-minion-role + environment: dev ''' from __future__ import absolute_import @@ -156,10 +158,12 @@ def ext_pillar(minion_id, path = comps[1].replace('root=', '') role = __salt__['grains.get']('role') + environment = __salt__['grains.get']('environment') # put the minion's ID in the path if necessary path %= { 'minion_id': minion_id, - 'role': role + 'role': role, + 'environment': environment } try: From 49d2e838cfee78c6cc748217afba6d5a6cf7ec88 Mon Sep 17 00:00:00 2001 From: Tim Egbert Date: Thu, 16 Jun 2016 09:20:41 -0600 Subject: [PATCH 449/769] Fixed a bug in the consul.py module that was preventing services (#34051) from registering in consul.agent_service_register. A nonexistent dict key was being referenced, causing an Excpetion and failure to register the service. This fix required only changing the offending line to use the "get" method and supplying a default value. --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 4bbf229..0e21208 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -84,7 +84,7 @@ def _query(function, ) if result.get('status', None) == salt.ext.six.moves.http_client.OK: - ret['data'] = result['dict'] + ret['data'] = result.get('dict', result) ret['res'] = True elif result.get('status', None) == salt.ext.six.moves.http_client.NO_CONTENT: ret['res'] = False From 9e77d4b8bc41a525def850b58a010269ec502ae1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Jun 2016 15:04:02 -0500 Subject: [PATCH 450/769] Revert py3modernize lint changes (#34339) * salt/__init__.py: revert py3modernize lint changes * salt/cloud/clouds/vmware.py: revert py3modernize lint changes * salt/modules/jboss7_cli.py: revert py3modernize lint changes * salt/payload.py: revert py3modernize lint changes * salt/serializers/yamlex.py: revert py3modernize lint changes * salt/states/win_servermanager.py: use absolute imports * salt/utils/args.py: revert py3modernize lint changes * salt/utils/decorators/__init__.py: use __name__ instead of func_name __name__ is py3-compatible and also works on builtins in py2, which func_name does not. * salt/utils/compat.py: revert py3modernize lint changes * salt/utils/__init__.py: revert py3modernize lint changes * salt/version.py: revert py3modernize lint changes * tests/salt-tcpdump.py: revert py3modernize lint changes --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f47130c..2e90399 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -60,7 +60,7 @@ def __define_global_system_encoding_variable__(): # than expected. See: # https://github.com/saltstack/salt/issues/21036 if sys.version_info[0] < 3: - import __builtin__ as builtins + import __builtin__ as builtins # pylint: disable=incompatible-py3-code else: import builtins # pylint: disable=import-error From 461b6f216787365aa33caf0b56fb4cd4a6b64a14 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Jun 2016 15:30:18 -0500 Subject: [PATCH 451/769] Remove repr formatting flag in places where it is used solely for quoting (#34183) * salt/cloud/__init__.py: remove repr formatting * salt/cloud/clouds/azurearm.py: remove repr formatting * salt/cloud/clouds/ec2.py: remove repr formatting * salt/cloud/clouds/profitbricks.py: remove repr formatting * salt/loader.py: remove repr formatting * salt/modules/win_file.py: remove repr formatting * salt/modules/zypper.py: remove repr formatting * salt/pillar/consul_pillar.py: remove repr formatting * salt/renderers/pyobjects.py: remove repr formatting * salt/returners/sentry_return.py: remove repr formatting * salt/states/bower.py: remove repr formatting * salt/states/cabal.py: remove repr formatting * salt/states/cmd.py: remove repr formatting * salt/states/composer.py: remove repr formatting * salt/states/win_network.py: remove repr formatting * salt/states/eselect.py: remove repr formatting * salt/states/file.py: remove repr formatting * salt/states/htpasswd.py: remove repr formatting * salt/states/memcached.py: remove repr formatting * salt/states/npm.py: remove repr formatting * salt/states/pip_state.py: remove repr formatting * salt/states/pkg.py: remove repr formatting * salt/states/pkgrepo.py: remove repr formatting * salt/states/supervisord.py: remove repr formatting * salt/states/timezone.py: remove repr formatting * salt/states/virtualenv_mod.py: remove repr formatting * salt/states/dockerio.py: remove repr formatting * salt/states/win_system.py: remove repr formatting * salt/utils/nb_popen.py: remove repr formatting * salt/utils/cloud.py: remove repr formatting * Add pylint disable due to legit usage of repr flag See https://github.com/saltstack/salt-pylint/pull/6 * Fix composer tests These tests needed to be updated because quoting was changed in the state module in 9dc9146. There was an unnecessary !r used for the exception class there, which means that instead of the exception class being passed through the formatter and coming out with the equivalent value of err.__str__(), we get a repr'ed instance of the exception class (i.e. SaltException('',)) in the state output. The unit test was asserting that we have that repr'ed instance of SaltException in the output, a case of writing the test to confirm the badly-conceived output in the state. This has also been corrected. * salt/cloud/clouds/azurearm.py: lint fixes * salt/modules/boto_s3_bucket.py: lint fixes * salt/modules/minion.py: lint fixes * salt/modules/reg.py: lint fixes * salt/modules/testinframod.py: lint fixes * salt/modules/win_iis.py: lint fixes * salt/pillar/csvpillar.py: lint fixes * salt/utils/win_functions.py: lint fixes * salt/states/nxos.py: lint fixes * salt/returners/mongo_future_return.py: lint fixes * tests/integration/__init__.py: lint fixes * tests/unit/context_test.py: lint fixes * tests/integration/states/file.py: lint fixes * tests/integration/utils/test_reactor.py: lint fixes * tests/integration/utils/testprogram.py: lint fixes * tests/unit/__init__.py: lint fixes * tests/integration/shell/minion.py: lint fixes * tests/unit/modules/boto_apigateway_test.py: lint fixes * tests/unit/modules/boto_cognitoidentity_test.py: lint fixes * tests/unit/modules/boto_elasticsearch_domain_test.py: lint fixes * tests/unit/modules/k8s_test.py: lint fixes * tests/unit/modules/reg_win_test.py: lint fixes * tests/unit/states/boto_apigateway_test.py: lint fixes * tests/unit/states/boto_cognitoidentity_test.py: lint fixes * tests/unit/states/boto_elasticsearch_domain_test.py: lint fixes --- src/saltext/consul/pillar/consul_pillar.py | 28 +++++++++------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index bd367de..392a22d 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -169,8 +169,7 @@ def ext_pillar(minion_id, try: pillar = fetch_tree(client, path) except KeyError: - log.error('No such key in consul profile {0}: {1}' - .format(profile, path)) + log.error('No such key in consul profile %s: %s', profile, path) pillar = {} return pillar @@ -199,8 +198,8 @@ def fetch_tree(client, path): return ret for item in reversed(items): key = re.sub(r'^' + path + '/?', '', item['Key']) - if key != "": - log.debug('key/path - {0}: {1}'.format(path, key)) + if key != '': + log.debug('key/path - %s: %s', path, key) log.debug('has_children? %r', format(has_children.search(key))) if has_children.search(key) is None: ret = pillar_format(ret, key.split('/'), item['Value']) @@ -282,20 +281,16 @@ def _resolve_datacenter(dc, pillarenv): If none patterns matched return ``None`` which meanse us datacenter of conencted Consul agent. ''' - log.debug("Resolving Consul datacenter based on: {dc}".format(dc=dc)) + log.debug('Resolving Consul datacenter based on: %s', dc) try: mappings = dc.items() # is it a dict? except AttributeError: - log.debug('Using pre-defined DC: {dc!r}'.format(dc=dc)) + log.debug('Using pre-defined DC: \'%s\'', dc) return dc - log.debug( - 'Selecting DC based on pillarenv using {num} pattern(s)'.format( - num=len(mappings) - ) - ) - log.debug('Pillarenv set to {env!r}'.format(env=pillarenv)) + log.debug('Selecting DC based on pillarenv using %d pattern(s)', len(mappings)) + log.debug('Pillarenv set to \'%s\'', pillarenv) # sort in reverse based on pattern length # but use alphabetic order within groups of patterns of same length @@ -304,13 +299,12 @@ def _resolve_datacenter(dc, pillarenv): for pattern, target in sorted_mappings: match = re.match(pattern, pillarenv) if match: - log.debug('Matched pattern: {pattern!r}'.format(pattern=pattern)) + log.debug('Matched pattern: \'%s\'', pattern) result = target.format(**match.groupdict()) - log.debug('Resolved datacenter: {result!r}'.format(result=result)) + log.debug('Resolved datacenter: \'%s\'', result) return result log.debug( - 'None of following patterns matched pillarenv={env}: {lst}'.format( - env=pillarenv, lst=', '.join(repr(x) for x in mappings) - ) + 'None of following patterns matched pillarenv=%s: %s', + pillarenv, ', '.join(repr(x) for x in mappings) ) From c4353ed38034cce9a6962ba279704267a2684745 Mon Sep 17 00:00:00 2001 From: Tim Egbert Date: Fri, 12 Aug 2016 11:07:16 -0600 Subject: [PATCH 452/769] Fixes consul.agent_service_register which was broken for registering service checks. The function "agent_service_register" in the consul.py module failed to register service health checks and did not follow the consul.io documentation for doing so. This patch fixes the code to successfully register service checks, check for dict keys case-insensitive (because the absense of keys with the correct case would fail silently), and more closely follow the documentation at consul.io, see: https://www.consul.io/docs/agent/http/agent.html#agent_service_register Here is an example salt state for registering a service and a couple of health checks, similar to the example in the consul.io documents: example-service-registration: module.run: - name: consul.agent_service_register - consul_url: "http://localhost:8500" - kwargs: id: redis1 name: "redis" tags: [master, v1] address: "127.0.0.1" port: 8000 EnableTagOverride: false check: script: "/usr/local/bin/check_redis.py" http: "http://localhost:5000/health" interval: "15s" NOTE: the saltstack documentation needs to be updated. It wasn't correct anyway. --- src/saltext/consul/modules/consul.py | 71 +++++++++++++++++----------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 0e21208..63931c0 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -12,6 +12,7 @@ # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urljoin as _urljoin +from salt.ext.six import iteritems import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module @@ -973,48 +974,62 @@ def agent_service_register(consul_url=None, **kwargs): ret['res'] = False return ret - if 'name' in kwargs: - data['Name'] = kwargs['name'] + lc_kwargs = dict() + for k, v in iteritems(kwargs): + lc_kwargs[k.lower()] = v + + if 'name' in lc_kwargs: + data['Name'] = lc_kwargs['name'] else: raise SaltInvocationError('Required argument "name" is missing.') - if 'address' in kwargs: - data['Address'] = kwargs['address'] + if 'address' in lc_kwargs: + data['Address'] = lc_kwargs['address'] - if 'port' in kwargs: - data['Port'] = kwargs['port'] + if 'port' in lc_kwargs: + data['Port'] = lc_kwargs['port'] - if 'id' in kwargs: - data['ID'] = kwargs['id'] + if 'id' in lc_kwargs: + data['ID'] = lc_kwargs['id'] - if 'tags' in kwargs: - _tags = kwargs['tags'] + if 'tags' in lc_kwargs: + _tags = lc_kwargs['tags'] if not isinstance(_tags, list): _tags = [_tags] data['Tags'] = _tags - check_elements = ('check_script', 'check_http', 'check_ttl') - if True in [True for item in check_elements if item in kwargs]: - data['Check'] = {} - - if 'check_script' in kwargs: - if 'interval' not in kwargs: + if 'enabletagoverride' in lc_kwargs: + data['EnableTagOverride'] = lc_kwargs['enabletagoverride'] + + if 'check' in lc_kwargs: + dd = dict() + for k, v in iteritems(lc_kwargs['check']): + dd[k.lower()] = v + interval_required = False + check_dd = dict() + + if 'script' in dd: + interval_required = True + check_dd['Script'] = dd['script'] + if 'http' in dd: + interval_required = True + check_dd['HTTP'] = dd['http'] + if 'ttl' in dd: + check_dd['TTL'] = dd['ttl'] + if 'interval' in dd: + check_dd['Interval'] = dd['interval'] + + if interval_required: + if 'Interval' not in check_dd: ret['message'] = 'Required parameter "interval" is missing.' ret['res'] = False return ret - data['Check']['Script'] = kwargs['check_script'] - data['Check']['Interval'] = kwargs['check_interval'] - - if 'check_ttl' in kwargs: - data['Check']['TTL'] = kwargs['check_ttl'] + else: + if 'Interval' in check_dd: + del check_dd['Interval'] # not required, so ignore it - if 'check_http' in kwargs: - if 'interval' not in kwargs: - ret['message'] = 'Required parameter "interval" is missing.' - ret['res'] = False - return ret - data['Check']['HTTP'] = kwargs['check_http'] - data['Check']['Interval'] = kwargs['check_interval'] + if len(check_dd) > 0: + data['Check'] = check_dd # if empty, ignore it function = 'agent/service/register' res = _query(consul_url=consul_url, From 8c959a83897a40c2263f7a7da2d0bc32d9fd2bfb Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Thu, 11 Aug 2016 18:00:45 +0300 Subject: [PATCH 453/769] Modular minion data cache. Grains, pillar and mine data are now managed by salt.cache subsystem --- src/saltext/consul/cache/__init__.py | 158 +++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 22 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 8afed05..bb337d6 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -8,18 +8,47 @@ import os import time from salt.loader import LazyLoader +from salt.payload import Serial class Cache(object): ''' - Main caching object + Base caching object providing access to the modular cache subsystem. + + Related configuration options: + + :param cache: + The name of the cache driver to use. This is the name of the python + module of the `salt.cache` package. Defult is `localfs`. + + :param serial: + The module of `salt.serializers` package that should be used by the cache + driver to store data. + If a driver can't use a specific module or uses specific objects storage + it can ignore this parameter. + + Terminology. + + Salt cache subsystem is organized as a tree with nodes and leafs like a + filesystem. Cache consists of banks. Each bank can contain a number of + keys. Each key can contain a dict or any other object serializable with + `salt.payload.Serial`. I.e. any data object in the cache can be + addressed by the path to the bank and the key name: + bank: 'minions/alpha' + key: 'data' + + Bank names should be formatted in a way that can be used as a + directory structure. If slashes are included in the name, then they + refer to a nested structure. + + Key name is a string identifier of a data container (like a file inside a + directory) which will hold the data. ''' - def __init__(self, opts, driver=None): + def __init__(self, opts): self.opts = opts + self.driver = opts['cache'] + self.serial = Serial(opts) self.modules = self._modules() - if driver is None: - driver = 'msgpack' - self.driver = driver def _modules(self, functions=None, whitelist=None): ''' @@ -32,7 +61,8 @@ def _modules(self, functions=None, whitelist=None): tag='cache', pack={ '__opts__': self.opts, - '__cache__': functions + '__cache__': functions, + '__context__': {'serial': self.serial}, }, whitelist=whitelist, ) @@ -79,23 +109,22 @@ def store(self, bank, key, data): ''' Store data using the specified module - bank + :param bank: The name of the location inside the cache which will hold the key and its associated data. - Bank names should be formatted in a way that can be used as a - directory structure. If slashes are included in the name, then they - refer to a nested directory structure (meaning, directories will be - created to accomodate the name). - - key + :param key: The name of the key (or file inside a directory) which will hold the data. File extensions should not be provided, as they will be added by the driver itself. - data + :param data: The data which will be stored in the cache. This data should be in a format which can be serialized by msgpack/json/yaml/etc. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'store') return self.modules[fun](bank, key, data) @@ -104,26 +133,111 @@ def fetch(self, bank, key): ''' Fetch data using the specified module - bank + :param bank: The name of the location inside the cache which will hold the key and its associated data. - Bank names should be formatted in a way that can be used as a - directory structure. If slashes are included in the name, then they - refer to a nested directory structure (meaning, directories will be - created to accomodate the name). - - key + :param key: The name of the key (or file inside a directory) which will hold the data. File extensions should not be provided, as they will be added by the driver itself. + + :return: + Return a python object fetched from the cache or None if the given + path or key not found. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'fetch') return self.modules[fun](bank, key) def updated(self, bank, key): ''' - Return the last updated epoch for the specified key + Get the last updated epoch for the specified key + + :param bank: + The name of the location inside the cache which will hold the key + and its associated data. + + :param key: + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + + :return: + Return an int epoch time in seconds or None if the object wasn't + found in cache. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'updated') return self.modules[fun](bank, key) + + def flush(self, bank, key=None): + ''' + Remove the key from the cache bank with all the key content. If no key is specified remove + the entire bank with all keys and sub-banks inside. + + :param bank: + The name of the location inside the cache which will hold the key + and its associated data. + + :param key: + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). + ''' + fun = '{0}.{1}'.format(self.driver, 'flush') + return self.modules[fun](bank) + + def list(self, bank): + ''' + Lists entries stored in the specified bank. + + :param bank: + The name of the location inside the cache which will hold the key + and its associated data. + + :return: + An iterable object containing all bank entries. Returns an empty + iterator if the bank doesn't exists. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). + ''' + fun = '{0}.{1}'.format(self.driver, 'list') + return self.modules[fun](bank) + + def contains(self, bank, key=None): + ''' + Checks if the specified bank contains the specified key. + + :param bank: + The name of the location inside the cache which will hold the key + and its associated data. + + :param key: + The name of the key (or file inside a directory) which will hold + the data. File extensions should not be provided, as they will be + added by the driver itself. + + :return: + Returns True if the specified key exists in the given bank and False + if not. + If key is None checks for the bank existense. + + :raises SaltCacheError: + Raises an exception if cache driver detected an error accessing data + in the cache backend (auth, permissions, etc). + ''' + fun = '{0}.{1}'.format(self.driver, 'contains') + return self.modules[fun](bank, key) From 06ded9daee6050ba5a1a28d10a98601c0dd6d9f1 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 17 Aug 2016 19:10:29 +0900 Subject: [PATCH 454/769] Fix import --- src/saltext/consul/modules/consul.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 63931c0..d8d16ef 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -12,7 +12,7 @@ # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urljoin as _urljoin -from salt.ext.six import iteritems +import salt.ext.six import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module @@ -975,7 +975,7 @@ def agent_service_register(consul_url=None, **kwargs): return ret lc_kwargs = dict() - for k, v in iteritems(kwargs): + for k, v in salt.ext.six.iteritems(kwargs): lc_kwargs[k.lower()] = v if 'name' in lc_kwargs: @@ -1003,7 +1003,7 @@ def agent_service_register(consul_url=None, **kwargs): if 'check' in lc_kwargs: dd = dict() - for k, v in iteritems(lc_kwargs['check']): + for k, v in salt.ext.six.iteritems(lc_kwargs['check']): dd[k.lower()] = v interval_required = False check_dd = dict() From f60ca4da63fa74c07f86681d400009c10afe9e8e Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 17 Aug 2016 17:06:44 +0300 Subject: [PATCH 455/769] Lazy init for cache lazyloader to prevent serialization on fork. --- src/saltext/consul/cache/__init__.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index bb337d6..5778b08 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -6,8 +6,8 @@ ''' from __future__ import absolute_import import os +import salt.loader import time -from salt.loader import LazyLoader from salt.payload import Serial @@ -48,24 +48,13 @@ def __init__(self, opts): self.opts = opts self.driver = opts['cache'] self.serial = Serial(opts) - self.modules = self._modules() + self._modules = None - def _modules(self, functions=None, whitelist=None): - ''' - Lazy load the cache modules - ''' - codedir = os.path.dirname(os.path.realpath(__file__)) - return LazyLoader( - [codedir], - self.opts, - tag='cache', - pack={ - '__opts__': self.opts, - '__cache__': functions, - '__context__': {'serial': self.serial}, - }, - whitelist=whitelist, - ) + @property + def modules(self): + if self._modules is None: + self._modules = salt.loader.cache(self.opts, self.serial) + return self._modules def cache(self, bank, key, fun, loop_fun=None, **kwargs): ''' From 4e81f7f96307292308a1baa51e99734db13da1eb Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 17 Aug 2016 11:18:49 -0600 Subject: [PATCH 456/769] Whitespace fix --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index d8d16ef..0c5e3e6 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -12,7 +12,7 @@ # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urljoin as _urljoin -import salt.ext.six +import salt.ext.six import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module From 253c42e578a23c16b1935a60950c21891bc9e61c Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Wed, 17 Aug 2016 15:33:17 -0600 Subject: [PATCH 457/769] Whitespace fix for 2015.8 (#35540) --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index d8d16ef..0c5e3e6 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -12,7 +12,7 @@ # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urljoin as _urljoin -import salt.ext.six +import salt.ext.six import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module From cda44dc48d5b49c6b9576b3f84d922a483a4aeba Mon Sep 17 00:00:00 2001 From: Jonathan Ballet Date: Tue, 6 Sep 2016 18:43:04 +0200 Subject: [PATCH 458/769] consul: fix formatting of consul.agent_join (#36061) --- src/saltext/consul/modules/consul.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 0c5e3e6..73b661c 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -609,11 +609,10 @@ def agent_join(consul_url=None, address=None, **kwargs): query_params=query_params) if res['res']: ret['res'] = True - ret['message'] = ('Agent maintenance mode ' - '{0}ed.'.format(kwargs['enable'])) + ret['message'] = 'Agent joined the cluster' else: ret['res'] = False - ret['message'] = 'Unable to change maintenance mode for agent.' + ret['message'] = 'Unable to join the cluster.' return ret From d0975b0c10794ab339599ae5eb34c58609ba2b83 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 13 Sep 2016 11:03:47 -0600 Subject: [PATCH 459/769] Clean up imports in salt/cache/__init__.py the 'os' import is not used, and splitting them out into python vs. salt imports makes them easier to read and more consistent with other salt files --- src/saltext/consul/cache/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 5778b08..da66ac9 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -2,12 +2,15 @@ ''' Loader mechanism for caching data, with data expirations, etc. -.. versionadded:: carbon +.. versionadded:: Carbon ''' + +# Import Python libs from __future__ import absolute_import -import os -import salt.loader import time + +# Import Salt lobs +import salt.loader from salt.payload import Serial From c3e432db684f29c3b939ec84f552b1542cac2fd5 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 13 Sep 2016 11:05:13 -0600 Subject: [PATCH 460/769] Pass the key kwarg through to the flush function --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index da66ac9..fdc52a8 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -188,7 +188,7 @@ def flush(self, bank, key=None): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'flush') - return self.modules[fun](bank) + return self.modules[fun](bank, key=key) def list(self, bank): ''' From 423b0e73ef5758a5f2b0705f9c4261c766343c7f Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Tue, 20 Sep 2016 09:31:44 +0200 Subject: [PATCH 461/769] Added "none" option for pillar_source_merging_strategy --- src/saltext/consul/pillar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c5a504f..26c0962 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -265,6 +265,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -378,6 +379,9 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): + if self.opts['pillar_source_merging_strategy'].lower() == "none": + if saltenv is not self.saltenv: + continue top = self.client.cache_file( self.opts['state_top'], saltenv From 99d573bc415717e0ed244ab10dbd655e2850b5c8 Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Tue, 20 Sep 2016 13:16:14 +0200 Subject: [PATCH 462/769] fix for when pillar_source_merging_strategy is not set --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 26c0962..e9ad6c0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -379,7 +379,7 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): - if self.opts['pillar_source_merging_strategy'].lower() == "none": + if self.opts.get('pillar_source_merging_strategy', None) == "none": if saltenv is not self.saltenv: continue top = self.client.cache_file( From 240d0e5defa599f83925c6c470c0a2001d0b6e5c Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Tue, 20 Sep 2016 14:48:25 +0200 Subject: [PATCH 463/769] Fix looking up pillar data when merging is set to "none" and no saltenv was passed --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e9ad6c0..d608a13 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -380,7 +380,7 @@ def get_tops(self): else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": - if saltenv is not self.saltenv: + if self.saltenv and saltenv is not self.saltenv: continue top = self.client.cache_file( self.opts['state_top'], From 0406c7f8b8552c246131ff784b2c05560da87922 Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Tue, 20 Sep 2016 15:08:47 +0200 Subject: [PATCH 464/769] Fix looking up pillar data when merging is set to "none" and no saltenv was passed retry --- src/saltext/consul/pillar/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d608a13..6725acb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -379,8 +379,10 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): - if self.opts.get('pillar_source_merging_strategy', None) == "none": - if self.saltenv and saltenv is not self.saltenv: + if self.opts['pillar_source_merging_strategy'] == "none": + if self.saltenv and saltenv != self.saltenv: + continue + if not self.saltenv and not saltenv == 'base': continue top = self.client.cache_file( self.opts['state_top'], From 0265802bf1ab6b3b32736fcbfa46be16d9e6061e Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Wed, 21 Sep 2016 08:40:47 +0200 Subject: [PATCH 465/769] Let's not break when no merging strategy is passed --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6725acb..ef8d9ab 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -379,7 +379,7 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): - if self.opts['pillar_source_merging_strategy'] == "none": + if self.opts.get('pillar_source_merging_strategy', None) == "none": if self.saltenv and saltenv != self.saltenv: continue if not self.saltenv and not saltenv == 'base': From 7e89c00fbe19f2dc1c6e328750e85b483b074634 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Fri, 23 Sep 2016 20:31:17 +0900 Subject: [PATCH 466/769] Cache the renderer in the pillar Prior to this, both salt-call and MWorkers would hit the disk to pull all of the renderers on each pillar generation. This should provide a marginal peformance increase for both. --- src/saltext/consul/pillar/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c5a504f..37f0e7a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -32,7 +32,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar=None, pillarenv=None, rend=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -48,7 +48,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar=pillar, pillarenv=pillarenv, rend=rend) # TODO: migrate everyone to this one! @@ -123,7 +123,7 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar=None, pillarenv=None, *args, **kwargs): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -259,7 +259,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar=None, pillarenv=None, rend=None): self.minion_id = minion_id # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] @@ -281,7 +281,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.functions = functions self.matcher = salt.minion.Matcher(self.opts, self.functions) - self.rend = salt.loader.render(self.opts, self.functions) + if rend is None: + self.rend = salt.loader.render(self.opts, self.functions) + else: + self.rend = rend ext_pillar_opts = copy.deepcopy(self.opts) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 From e2a49d0c3f7e3aea9f62c1153789d36d6bb6b4e8 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Fri, 23 Sep 2016 13:36:23 -0600 Subject: [PATCH 467/769] Back-port #36435 to 2016.3 (#36532) * Added "none" option for pillar_source_merging_strategy * added none to merging strategy in dictupdate so we no longer log warnings when it is selected * Updated documentation * fix for when pillar_source_merging_strategy is not set * Fix looking up pillar data when merging is set to "none" and no saltenv was passed * Fix looking up pillar data when merging is set to "none" and no saltenv was passed retry * Let's not break when no merging strategy is passed * Capitalisation * Update on doc --- src/saltext/consul/pillar/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7090191..ceaa4ae 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -279,6 +279,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) if opts.get('file_client', '') == 'local': @@ -399,6 +400,11 @@ def get_tops(self): ] else: for saltenv in self._get_envs(): + if self.opts.get('pillar_source_merging_strategy', None) == "none": + if self.saltenv and saltenv != self.saltenv: + continue + if not self.saltenv and not saltenv == 'base': + continue top = self.client.cache_file( self.opts['state_top'], saltenv From c6248f9d27d575c662c63516e98216d10ec4e1b4 Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Tue, 27 Sep 2016 09:05:52 -0600 Subject: [PATCH 468/769] Add support for ACL Tokens in consul_pillar with the option consul.token --- src/saltext/consul/pillar/consul_pillar.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index b84b91c..6ec359c 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -12,6 +12,9 @@ my_consul_config: consul.host: 127.0.0.1 consul.port: 8500 + consul.token: my_consul_acl_token + +The ``consul.token`` is optional and requires python-consul >= 0.4.7. After the profile is created, configure the external pillar system to use it. Optionally, a root may be specified. @@ -64,6 +67,7 @@ try: import consul HAS_CONSUL = True + CONSUL_VERSION = consul.__version__ except ImportError: HAS_CONSUL = False @@ -186,9 +190,14 @@ def get_conn(opts, profile): consul_host = conf.get('consul.host', '127.0.0.1') consul_port = conf.get('consul.port', 8500) + consul_token = conf.get('consul.token', None) if HAS_CONSUL: - return consul.Consul(host=consul_host, port=consul_port) + # Sanity check. ACL Tokens are supported on python-consul 0.4.7 onwards only. + if CONSUL_VERSION >= '0.4.7': + return consul.Consul(host=consul_host, port=consul_port, token=consul_token) + else: + return consul.Consul(host=consul_host, port=consul_port) else: raise CommandExecutionError( '(unable to import consul, ' From 96514f87bc8b9ce83e8c90298f0fa625fc1f7cc2 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 28 Sep 2016 08:12:34 -0600 Subject: [PATCH 469/769] Add consul host, port, and token values back in with conf.get() --- src/saltext/consul/pillar/consul_pillar.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 545506a..a6f4275 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -114,9 +114,7 @@ # Import python libs import logging - import re - import yaml from salt.exceptions import CommandExecutionError @@ -261,6 +259,10 @@ def get_conn(opts, profile): pillarenv = opts_merged.get('pillarenv') or 'base' params['dc'] = _resolve_datacenter(params['dc'], pillarenv) + consul_host = params.get('host') + consul_port = params.get('port') + consul_token = params.get('token') + if HAS_CONSUL: # Sanity check. ACL Tokens are supported on python-consul 0.4.7 onwards only. if CONSUL_VERSION >= '0.4.7': From 6a06c56e11760fc75d06f18a24f395db265b3cd4 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 28 Sep 2016 08:36:11 -0600 Subject: [PATCH 470/769] Better merge conflict fix: pass **params to consul.Consul in consul_pillar.py --- src/saltext/consul/pillar/consul_pillar.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index a6f4275..7d3d366 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -259,16 +259,11 @@ def get_conn(opts, profile): pillarenv = opts_merged.get('pillarenv') or 'base' params['dc'] = _resolve_datacenter(params['dc'], pillarenv) - consul_host = params.get('host') - consul_port = params.get('port') - consul_token = params.get('token') - if HAS_CONSUL: # Sanity check. ACL Tokens are supported on python-consul 0.4.7 onwards only. - if CONSUL_VERSION >= '0.4.7': - return consul.Consul(host=consul_host, port=consul_port, token=consul_token) - else: - return consul.Consul(host=consul_host, port=consul_port) + if CONSUL_VERSION < '0.4.7' and params.get('target'): + params.pop('target') + return consul.Consul(**params) else: raise CommandExecutionError( '(unable to import consul, ' From 50bb7796a1d4c06197764a78d015a45eddbabe72 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Oct 2016 15:44:04 -0500 Subject: [PATCH 471/769] Fix pillar merging when ext_pillar_first is enabled ext_pillar was being merged into pillar, when it should have been the other way around. This means that when ext_pillar_first was enabled, pillar keys with the same name as ones defined in ext_pillar were being lost instead of overriding ext_pillar. --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 578b4df..e0c6555 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -791,8 +791,8 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) - pillar = merge(pillar, - self.opts['pillar'], + pillar = merge(self.opts['pillar'], + pillar, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) From bdc3a997322b9eebee602125867244b9c40f4cdb Mon Sep 17 00:00:00 2001 From: Anbang Wen Date: Tue, 25 Oct 2016 12:59:57 +0100 Subject: [PATCH 472/769] decode pillar on minion side on receiving Fix a set of encoding issue when non-ascii is used in pillar, e.g. #3436, #32975 --- src/saltext/consul/pillar/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 2b0b2ca..224a325 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -24,6 +24,7 @@ from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict from salt.version import __version__ +from salt.utils.locales import decode_recursively # Import 3rd-party libs import salt.ext.six as six @@ -162,7 +163,7 @@ def compile_pillar(self): '{1}'.format(type(ret_pillar).__name__, ret_pillar) ) return {} - return ret_pillar + return decode_recursively(ret_pillar) class PillarCache(object): From c770864c17db6e2425a8162bed53da9b25b4de14 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 27 Oct 2016 15:00:00 -0600 Subject: [PATCH 473/769] Update Carbon versionadded tags to 2016.11.0 in salt/* --- src/saltext/consul/cache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index fdc52a8..3499b94 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- ''' -Loader mechanism for caching data, with data expirations, etc. +Loader mechanism for caching data, with data expiration, etc. -.. versionadded:: Carbon +.. versionadded:: 2016.11.0 ''' # Import Python libs From 15f76d309a3ee9af584cf95bcf2545ecb1a3e957 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 29 Oct 2016 20:21:48 -0700 Subject: [PATCH 474/769] Adding some fixes to the console module to address #36994 & #36995 --- src/saltext/consul/modules/consul.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 73b661c..a5da2c8 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -253,13 +253,21 @@ def put(consul_url=None, key=None, value=None, **kwargs): if not key: raise SaltInvocationError('Required argument "key" is missing.') + # Invalid to specified these together + conflicting_args = ['cas', 'release', 'acquire'] + for _l1 in conflicting_args: + for _l2 in conflicting_args: + if _l1 in kwargs and _l2 in kwargs and _l1 != _l2: + raise SaltInvocationError('Using arguments `{0}` and `{1}`' + ' together is invalid.'.format(_l1, _l2)) + query_params = {} available_sessions = session_list(consul_url=consul_url, return_list=True) _current = get(consul_url=consul_url, key=key) if 'flags' in kwargs: - if not kwargs['flags'] >= 0 and not kwargs['flags'] <= 2**64: + if kwargs['flags'] >= 0 and kwargs['flags'] <= 2**64: query_params['flags'] = kwargs['flags'] if 'cas' in kwargs: @@ -281,8 +289,6 @@ def put(consul_url=None, key=None, value=None, **kwargs): 'CAS argument can not be used.'.format(key)) ret['res'] = False return ret - else: - log.error('Key {0} does not exist. Skipping release.') if 'acquire' in kwargs: if kwargs['acquire'] not in available_sessions: From 766f55f9eae0f50465584c47ad6fc819015dc9b4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 16 Nov 2016 01:10:54 -0600 Subject: [PATCH 475/769] Fix for pillar setting 'environment' key in __gen_opts() 7a6e402 added ``environment`` as a key in the master opts. This had the unfortunate side effect of breaking the dynamic pillar feature in git_pillar, which relies on pillar setting this opts key to the value from the minion opts. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f75aaec..35faa3c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -348,7 +348,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren opts['grains'] = {} else: opts['grains'] = grains - if 'environment' not in opts: + if not opts.get('environment'): opts['environment'] = saltenv opts['id'] = self.minion_id if 'pillarenv' not in opts: From 8cd1e2f6f95130f7adb6944dc3c546aba2195289 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 29 Nov 2016 15:35:04 -0600 Subject: [PATCH 476/769] Revert addition of pillar_roots_override_ext_pillar This turned out to be an unnecessary addition, as the root cause was ext_pillar_first not behaving as documented. --- src/saltext/consul/pillar/__init__.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bf2508c..7821387 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -780,27 +780,16 @@ def compile_pillar(self, ext=True, pillar_dirs=None): ''' top, top_errors = self.get_top() if ext: - if self.opts.get('pillar_roots_override_ext_pillar', False) or self.opts.get('ext_pillar_first', False): - salt.utils.warn_until('Nitrogen', - 'The \'ext_pillar_first\' option has been deprecated and ' - 'replaced by \'pillar_roots_override_ext_pillar\'.' - ) + if self.opts.get('ext_pillar_first', False): self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) - if self.opts.get('pillar_roots_override_ext_pillar', False): - pillar = merge(self.opts['pillar'], - pillar, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) - else: - pillar = merge(pillar, - self.opts['pillar'], - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge(self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) From 0e5c3a3ed864aa30154c1b06f2ffc5b0aa8c5d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Wed, 30 Nov 2016 12:47:37 +0100 Subject: [PATCH 477/769] consul_pillar support for limiting pillar exposure via minion targeting --- src/saltext/consul/pillar/consul_pillar.py | 54 +++++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 7d3d366..fbc4453 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -109,6 +109,16 @@ role: my-minion-role environment: dev +It's possible to lock down where the pillar values are shared through minion +targeting. Note that double quotes ``"`` are required around the target value +and cannot be used inside the matching statement. See the section on Compound +Matchers for more examples. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config root=salt target="L@salt.example.com and G@osarch:x86_64" + ''' from __future__ import absolute_import @@ -116,6 +126,7 @@ import logging import re import yaml +import salt.utils.minions from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge @@ -147,30 +158,49 @@ def ext_pillar(minion_id, ''' Check consul for all data ''' - comps = conf.split() - - profile = None - if comps[0]: - profile = comps[0] - client = get_conn(__opts__, profile) + opts = {} + temp = conf + target_re = re.compile('target="(.*?)"') + match = target_re.search(temp) + if match: + opts['target'] = match.group(1) + temp = temp.strip(match.group(0)) + checker = salt.utils.minions.CkMinions(__opts__) + minions = checker.check_minions(opts['target'], 'compound') + if minion_id not in minions: + return {} + + root_re = re.compile('root=(\S*)') + match = root_re.search(temp) + if match: + opts['root'] = match.group(1) + temp = temp.strip(match.group(0)) + else: + opts['root'] = "" - path = '' - if len(comps) > 1 and comps[1].startswith('root='): - path = comps[1].replace('root=', '') + profile_re = re.compile('(?:profile=)?(\S+)') + match = profile_re.search(temp) + if match: + opts['profile'] = match.group(1) + temp = temp.strip(match.group(0)) + else: + opts['profile'] = None + + client = get_conn(__opts__, opts['profile']) role = __salt__['grains.get']('role') environment = __salt__['grains.get']('environment') # put the minion's ID in the path if necessary - path %= { + opts['root'] %= { 'minion_id': minion_id, 'role': role, 'environment': environment } try: - pillar = fetch_tree(client, path) + pillar = fetch_tree(client, opts['root']) except KeyError: - log.error('No such key in consul profile %s: %s', profile, path) + log.error('No such key in consul profile %s: %s', opts['profile'], opts['root']) pillar = {} return pillar From 6aa9fbda702c644fc7d794662c3ffc42af2a3a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Fri, 2 Dec 2016 10:47:26 +0100 Subject: [PATCH 478/769] Fixed possible incorrect behavior if target wasn't on start/end of str --- src/saltext/consul/pillar/consul_pillar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index fbc4453..c36cc43 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -164,7 +164,7 @@ def ext_pillar(minion_id, match = target_re.search(temp) if match: opts['target'] = match.group(1) - temp = temp.strip(match.group(0)) + temp = temp.replace(match.group(0), '') checker = salt.utils.minions.CkMinions(__opts__) minions = checker.check_minions(opts['target'], 'compound') if minion_id not in minions: @@ -174,7 +174,7 @@ def ext_pillar(minion_id, match = root_re.search(temp) if match: opts['root'] = match.group(1) - temp = temp.strip(match.group(0)) + temp = temp.replace(match.group(0), '') else: opts['root'] = "" @@ -182,7 +182,7 @@ def ext_pillar(minion_id, match = profile_re.search(temp) if match: opts['profile'] = match.group(1) - temp = temp.strip(match.group(0)) + temp = temp.replace(match.group(0), '') else: opts['profile'] = None From 1c567d011d04a3f48227f7276c3c098d3495f6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Mon, 5 Dec 2016 11:13:45 +0100 Subject: [PATCH 479/769] Linting fixes --- src/saltext/consul/pillar/consul_pillar.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index c36cc43..cb302dd 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -163,29 +163,29 @@ def ext_pillar(minion_id, target_re = re.compile('target="(.*?)"') match = target_re.search(temp) if match: - opts['target'] = match.group(1) - temp = temp.replace(match.group(0), '') + opts['target'] = match.group(1) + temp = temp.replace(match.group(0), '') checker = salt.utils.minions.CkMinions(__opts__) minions = checker.check_minions(opts['target'], 'compound') if minion_id not in minions: - return {} + return {} root_re = re.compile('root=(\S*)') match = root_re.search(temp) if match: - opts['root'] = match.group(1) - temp = temp.replace(match.group(0), '') + opts['root'] = match.group(1) + temp = temp.replace(match.group(0), '') else: - opts['root'] = "" + opts['root'] = "" profile_re = re.compile('(?:profile=)?(\S+)') match = profile_re.search(temp) if match: - opts['profile'] = match.group(1) - temp = temp.replace(match.group(0), '') + opts['profile'] = match.group(1) + temp = temp.replace(match.group(0), '') else: - opts['profile'] = None - + opts['profile'] = None + client = get_conn(__opts__, opts['profile']) role = __salt__['grains.get']('role') From 8f2bc96e8ededb87196645d75139b7ec00649960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Bj=C3=B6rklin?= Date: Mon, 5 Dec 2016 13:41:47 +0100 Subject: [PATCH 480/769] Ignore W1401 (anomalous-backslash-in-string) --- src/saltext/consul/pillar/consul_pillar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index cb302dd..0a4384c 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -170,7 +170,7 @@ def ext_pillar(minion_id, if minion_id not in minions: return {} - root_re = re.compile('root=(\S*)') + root_re = re.compile('root=(\S*)') # pylint: disable=W1401 match = root_re.search(temp) if match: opts['root'] = match.group(1) @@ -178,7 +178,7 @@ def ext_pillar(minion_id, else: opts['root'] = "" - profile_re = re.compile('(?:profile=)?(\S+)') + profile_re = re.compile('(?:profile=)?(\S+)') # pylint: disable=W1401 match = profile_re.search(temp) if match: opts['profile'] = match.group(1) From f4c1a9625bd77c8537b29469ba8028ebd6cdfb51 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 14 Dec 2016 12:13:07 -0600 Subject: [PATCH 481/769] Also check if pillarenv is in opts This will future-proof this code against a problem similar to the one fixed by https://github.com/saltstack/salt/pull/37721 --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 35faa3c..6caf38b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -351,7 +351,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren if not opts.get('environment'): opts['environment'] = saltenv opts['id'] = self.minion_id - if 'pillarenv' not in opts: + if not opts.get('pillarenv'): opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] From 7b39ed63f96b85d4c10a8b64a61d5350e26d6748 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Tue, 20 Dec 2016 19:32:37 +0300 Subject: [PATCH 482/769] Consul data cache plugin. --- src/saltext/consul/cache/consul.py | 131 +++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 src/saltext/consul/cache/consul.py diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py new file mode 100644 index 0000000..1e0fb38 --- /dev/null +++ b/src/saltext/consul/cache/consul.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +''' +Data cache plugin for Consul key/value data store. +This requires python-consul python package. + +.. versionadded:: 2016.11.2 +''' +from __future__ import absolute_import +import logging +try: + import consul + HAS_CONSUL = True +except ImportError: + HAS_CONSUL = False + +from salt.exceptions import SaltCacheError + +log = logging.getLogger(__name__) +CONSUL = None + + +# Define the module's virtual name +__virtualname__ = 'consul' + + +def __virtual__(): + ''' + Confirm this python-consul package is installed + ''' + if not HAS_CONSUL: + return (False, "Please install python-consul package to use consul data cache driver") + + global CONSUL + CONSUL = consul.Consul() + + return __virtualname__ + + +def store(bank, key, data): + ''' + Store a key value. + ''' + c_key = '{0}/{1}'.format(bank, key) + try: + c_data = __context__['serial'].dumps(data) + CONSUL.kv.put(c_key, c_data) + except Exception as exc: + raise SaltCacheError( + 'There was an error writing the key, {0}: {1}'.format( + c_key, exc + ) + ) + + +def fetch(bank, key): + ''' + Fetch a key value. + ''' + c_key = '{0}/{1}'.format(bank, key) + try: + _, value = CONSUL.kv.get(c_key) + if value is None: + return value + return __context__['serial'].loads(value['Value']) + except Exception as exc: + raise SaltCacheError( + 'There was an error reading the key, {0}: {1}'.format( + c_key, exc + ) + ) + + +def flush(bank, key=None): + ''' + Remove the key from the cache bank with all the key content. + ''' + if key is None: + c_key = bank + else: + c_key = '{0}/{1}'.format(bank, key) + try: + return CONSUL.kv.delete(c_key, recurse=key is None) + except Exception as exc: + raise SaltCacheError( + 'There was an error removing the key, {0}: {1}'.format( + c_key, exc + ) + ) + + +def list(bank): + ''' + Return an iterable object containing all entries stored in the specified bank. + ''' + try: + _, keys = CONSUL.kv.get(bank + '/', keys=True, separator='/') + except Exception as exc: + raise SaltCacheError( + 'There was an error getting the key "{0}": {1}'.format( + bank, exc + ) + ) + if keys is None: + keys = [] + else: + # Any key could be a branch and a leaf at the same time in Consul + # so we have to return a list of unique names only. + out = set() + for key in keys: + out.add(key[len(bank) + 1:].rstrip('/')) + keys = list(out) + return keys + + +def contains(bank, key): + ''' + Checks if the specified bank contains the specified key. + ''' + if key is None: + return True # any key could be a branch and a leaf at the same time in Consul + else: + try: + c_key = '{0}/{1}'.format(bank, key) + _, value = CONSUL.kv.get(c_key) + except Exception as exc: + raise SaltCacheError( + 'There was an error getting the key, {0}: {1}'.format( + c_key, exc + ) + ) + return value is not None From 1a289f350345546fc0574cc6b56f7f85c66f578b Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Tue, 20 Dec 2016 21:27:43 +0300 Subject: [PATCH 483/769] Configuration options and documentation for Consul data cache plugin. --- src/saltext/consul/cache/consul.py | 61 +++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 1e0fb38..fd9723e 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -1,7 +1,38 @@ # -*- coding: utf-8 -*- ''' -Data cache plugin for Consul key/value data store. -This requires python-consul python package. +Minion data cache plugin for Consul key/value data store. + +It is up to the system administrator to set up and configure the Consul +infrastructure. All is needed for this plugin is a working Consul agent +with a read-write access to the key-value storae. + +The related documentation can be found here: https://www.consul.io/docs/index.html + +To enable this cache plugin the master will need the python client for +Consul installed that could be easily done with `pip install python-consul`. + +Optionally depending on the Consul agent configuration the following values +could be set in the master config, these are the defaults: + +.. code-block:: yaml + + consul.host: 127.0.0.1 + consul.port: 8500 + consul.token: None + consul.scheme: http + consul.consistency: default + consul.dc: None + consul.verify: True + +Related docs could be found here: +* python-consul: https://python-consul.readthedocs.io/en/latest/#consul + +To use the consul as a minion data cache backend set the master `cache` config +value to `consul`: + +.. code-block:: yaml + + cache: consul .. versionadded:: 2016.11.2 ''' @@ -16,7 +47,7 @@ from salt.exceptions import SaltCacheError log = logging.getLogger(__name__) -CONSUL = None +api = None # Define the module's virtual name @@ -30,8 +61,18 @@ def __virtual__(): if not HAS_CONSUL: return (False, "Please install python-consul package to use consul data cache driver") - global CONSUL - CONSUL = consul.Consul() + consul_kwargs = { + 'host': __opts__.get('consul.host', '127.0.0.1'), + 'port': __opts__.get('consul.port', 8500), + 'token': __opts__.get('consul.token', None), + 'scheme': __opts__.get('consul.scheme', 'http'), + 'consistency': __opts__.get('consul.consistency', 'default'), + 'dc': __opts__.get('consul.dc', None), + 'verify': __opts__.get('consul.verify', True), + } + + global api + api = consul.Consul(**consul_kwargs) return __virtualname__ @@ -43,7 +84,7 @@ def store(bank, key, data): c_key = '{0}/{1}'.format(bank, key) try: c_data = __context__['serial'].dumps(data) - CONSUL.kv.put(c_key, c_data) + api.kv.put(c_key, c_data) except Exception as exc: raise SaltCacheError( 'There was an error writing the key, {0}: {1}'.format( @@ -58,7 +99,7 @@ def fetch(bank, key): ''' c_key = '{0}/{1}'.format(bank, key) try: - _, value = CONSUL.kv.get(c_key) + _, value = api.kv.get(c_key) if value is None: return value return __context__['serial'].loads(value['Value']) @@ -79,7 +120,7 @@ def flush(bank, key=None): else: c_key = '{0}/{1}'.format(bank, key) try: - return CONSUL.kv.delete(c_key, recurse=key is None) + return api.kv.delete(c_key, recurse=key is None) except Exception as exc: raise SaltCacheError( 'There was an error removing the key, {0}: {1}'.format( @@ -93,7 +134,7 @@ def list(bank): Return an iterable object containing all entries stored in the specified bank. ''' try: - _, keys = CONSUL.kv.get(bank + '/', keys=True, separator='/') + _, keys = api.kv.get(bank + '/', keys=True, separator='/') except Exception as exc: raise SaltCacheError( 'There was an error getting the key "{0}": {1}'.format( @@ -121,7 +162,7 @@ def contains(bank, key): else: try: c_key = '{0}/{1}'.format(bank, key) - _, value = CONSUL.kv.get(c_key) + _, value = api.kv.get(c_key) except Exception as exc: raise SaltCacheError( 'There was an error getting the key, {0}: {1}'.format( From 1ed4f4f029acbbba44ba1c450afa3be6ce15834e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 21 Dec 2016 19:50:08 -0600 Subject: [PATCH 484/769] Add pillarenv_from_saltenv to salt.pillar --- src/saltext/consul/pillar/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 784dc37..f15ddbb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -78,8 +78,12 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.grains = grains self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) - if pillarenv is not None or 'pillarenv' not in self.opts: + if pillarenv is not None: self.opts['pillarenv'] = pillarenv + elif self.opts.get('pillarenv_from_saltenv', False): + self.opts['pillarenv'] = saltenv + elif 'pillarenv' not in self.opts: + self.opts['pillarenv'] = None self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): @@ -131,8 +135,12 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.grains = grains self.minion_id = minion_id self.channel = salt.transport.Channel.factory(opts) - if pillarenv is not None or 'pillarenv' not in self.opts: + if pillarenv is not None: self.opts['pillarenv'] = pillarenv + elif self.opts.get('pillarenv_from_saltenv', False): + self.opts['pillarenv'] = saltenv + elif 'pillarenv' not in self.opts: + self.opts['pillarenv'] = None self.pillar_override = {} if pillar is not None: if isinstance(pillar, dict): From 4762d488e7fcd2336182c2b08dabc3db6f39bc24 Mon Sep 17 00:00:00 2001 From: Thilo Schmalfuss Date: Mon, 19 Dec 2016 14:13:56 +0100 Subject: [PATCH 485/769] ext-pillar nodegroups works for all minions now. --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 3499b94..e05035b 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -22,7 +22,7 @@ class Cache(object): :param cache: The name of the cache driver to use. This is the name of the python - module of the `salt.cache` package. Defult is `localfs`. + module of the `salt.cache` package. Default is `localfs`. :param serial: The module of `salt.serializers` package that should be used by the cache From b6baf46adc02ce53b5cf316aeb69e1e53148f1d2 Mon Sep 17 00:00:00 2001 From: Yoram Hekma Date: Wed, 11 Jan 2017 08:52:31 +0100 Subject: [PATCH 486/769] We do not want to overload the list() type because if we do, we turn this function into a recursive one, which results in an exception because set() cannot be concatenated with str ('/') --- src/saltext/consul/cache/__init__.py | 2 +- src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index e05035b..bd61f0c 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -206,7 +206,7 @@ def list(self, bank): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'list') + fun = '{0}.{1}'.format(self.driver, 'getlist') return self.modules[fun](bank) def contains(self, bank, key=None): diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index fd9723e..090bab3 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -129,7 +129,7 @@ def flush(bank, key=None): ) -def list(bank): +def getlist(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' From 56b51db0cb6cbb18722d838556f8925232f1d7b3 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 10 Jan 2017 09:10:47 -0700 Subject: [PATCH 487/769] Don't force salt.cache to use cachedir from opts --- src/saltext/consul/cache/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index bd61f0c..79f29e0 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -47,8 +47,12 @@ class Cache(object): Key name is a string identifier of a data container (like a file inside a directory) which will hold the data. ''' - def __init__(self, opts): + def __init__(self, opts, cachedir=None): self.opts = opts + if cachedir is None: + self.cachedir = opts['cachedir'] + else: + self.cachedir = cachedir self.driver = opts['cache'] self.serial = Serial(opts) self._modules = None @@ -119,7 +123,7 @@ def store(self, bank, key, data): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'store') - return self.modules[fun](bank, key, data) + return self.modules[fun](bank, key, data, self.cachedir) def fetch(self, bank, key): ''' @@ -143,7 +147,7 @@ def fetch(self, bank, key): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'fetch') - return self.modules[fun](bank, key) + return self.modules[fun](bank, key, self.cachedir) def updated(self, bank, key): ''' @@ -167,7 +171,7 @@ def updated(self, bank, key): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'updated') - return self.modules[fun](bank, key) + return self.modules[fun](bank, key, self.cachedir) def flush(self, bank, key=None): ''' @@ -188,7 +192,7 @@ def flush(self, bank, key=None): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'flush') - return self.modules[fun](bank, key=key) + return self.modules[fun](bank, key=key, cachedir=self.cachedir) def list(self, bank): ''' @@ -206,8 +210,8 @@ def list(self, bank): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'getlist') - return self.modules[fun](bank) + fun = '{0}.{1}'.format(self.driver, 'list') + return self.modules[fun](bank, self.cachedir) def contains(self, bank, key=None): ''' @@ -232,4 +236,4 @@ def contains(self, bank, key=None): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'contains') - return self.modules[fun](bank, key) + return self.modules[fun](bank, key, self.cachedir) From bf628bc26a54d9cd929eca473aeaf5dc98e47b22 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 10 Jan 2017 10:16:22 -0700 Subject: [PATCH 488/769] Not every module will need cachedir --- src/saltext/consul/cache/__init__.py | 30 ++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 79f29e0..b59ba46 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -123,7 +123,10 @@ def store(self, bank, key, data): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'store') - return self.modules[fun](bank, key, data, self.cachedir) + try: + return self.modules[fun](bank, key, data, self.cachedir) + except TypeError: + return self.modules[fun](bank, key, data) def fetch(self, bank, key): ''' @@ -147,7 +150,10 @@ def fetch(self, bank, key): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'fetch') - return self.modules[fun](bank, key, self.cachedir) + try: + return self.modules[fun](bank, key, self.cachedir) + except TypeError: + return self.modules[fun](bank, key) def updated(self, bank, key): ''' @@ -171,7 +177,10 @@ def updated(self, bank, key): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'updated') - return self.modules[fun](bank, key, self.cachedir) + try: + return self.modules[fun](bank, key, self.cachedir) + except TypeError: + return self.modules[fun](bank, key) def flush(self, bank, key=None): ''' @@ -192,7 +201,10 @@ def flush(self, bank, key=None): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'flush') - return self.modules[fun](bank, key=key, cachedir=self.cachedir) + try: + return self.modules[fun](bank, key=key, cachedir=self.cachedir) + except TypeError: + return self.modules[fun](bank, key=key) def list(self, bank): ''' @@ -211,7 +223,10 @@ def list(self, bank): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'list') - return self.modules[fun](bank, self.cachedir) + try: + return self.modules[fun](bank, self.cachedir) + except TypeError: + return self.modules[fun](bank) def contains(self, bank, key=None): ''' @@ -236,4 +251,7 @@ def contains(self, bank, key=None): in the cache backend (auth, permissions, etc). ''' fun = '{0}.{1}'.format(self.driver, 'contains') - return self.modules[fun](bank, key, self.cachedir) + try: + return self.modules[fun](bank, key, self.cachedir) + except TypeError: + return self.modules[fun](bank, key) From 98699a377f43844518155c9ccda28a52d20f4df3 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Tue, 10 Jan 2017 15:32:29 -0700 Subject: [PATCH 489/769] Default to CACHE_DIR in syspaths --- src/saltext/consul/cache/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index b59ba46..f90e867 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -11,6 +11,7 @@ # Import Salt lobs import salt.loader +import salt.syspaths from salt.payload import Serial @@ -50,7 +51,7 @@ class Cache(object): def __init__(self, opts, cachedir=None): self.opts = opts if cachedir is None: - self.cachedir = opts['cachedir'] + self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) else: self.cachedir = cachedir self.driver = opts['cache'] From 70edfa5291b8e088f040c9ed4b31d8084a250055 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Sun, 15 Jan 2017 14:52:17 -0700 Subject: [PATCH 490/769] Change getlist back to list (using _list) --- src/saltext/consul/cache/consul.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 090bab3..3a6afd2 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -46,6 +46,9 @@ from salt.exceptions import SaltCacheError +# Don't shadow built-ins +__func_alias__ = {'list_': 'list'} + log = logging.getLogger(__name__) api = None @@ -129,7 +132,7 @@ def flush(bank, key=None): ) -def getlist(bank): +def _list(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' @@ -153,6 +156,9 @@ def getlist(bank): return keys +getlist = _list + + def contains(bank, key): ''' Checks if the specified bank contains the specified key. From 7c5dd17761b751b78bf3a3cd46ff03e154503e72 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Mon, 16 Jan 2017 07:47:31 -0700 Subject: [PATCH 491/769] Typo fix --- src/saltext/consul/cache/consul.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 3a6afd2..5dee30b 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -132,7 +132,7 @@ def flush(bank, key=None): ) -def _list(bank): +def list_(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' @@ -156,7 +156,7 @@ def _list(bank): return keys -getlist = _list +getlist = list_ def contains(bank, key): From 3d3b378b24c5bb03475563497e6d370cab8dac0c Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Wed, 18 Jan 2017 09:36:55 -0500 Subject: [PATCH 492/769] Extend support for pillarenv_from_saltenv to master config `pillarenv_from_saltenv` is already an option for minion config, this change enables this feature on the master. If set to `true` salt will derive the pillar environment set on the master: $ salt-run pillar.show_pillar saltenv=radman my_passphrase: XYZ $ sudo salt-run pillar.show_pillar saltenv=radman2 my_passphrase: ABC --- src/saltext/consul/pillar/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f15ddbb..88aad09 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -270,6 +270,9 @@ class Pillar(object): def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None, rend=None): self.minion_id = minion_id + if pillarenv is None: + if opts.get('pillarenv_from_saltenv', False): + opts['pillarenv'] = saltenv # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client From ccfffe1560117dd1b58fdfe0aa067cb5264cecaa Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 19 Jan 2017 15:53:22 -0700 Subject: [PATCH 493/769] Add documentation for the Minion data cache Includes docs for localfs and consul minion data cache modules --- docs/ref/cache/all/salt.cache.consul.rst | 5 +++++ src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/ref/cache/all/salt.cache.consul.rst diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst new file mode 100644 index 0000000..516a3b8 --- /dev/null +++ b/docs/ref/cache/all/salt.cache.consul.rst @@ -0,0 +1,5 @@ +salt.cache.consul module +======================== + +.. automodule:: salt.cache.consul + :members: diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 090bab3..f89ad82 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -21,7 +21,7 @@ consul.token: None consul.scheme: http consul.consistency: default - consul.dc: None + consul.dc: dc1 consul.verify: True Related docs could be found here: From 8058c5b93fddd96f4a3741778251596db0e6d58b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 25 Jan 2017 14:37:16 -0600 Subject: [PATCH 494/769] Make on-demand ext_pillars tunable --- src/saltext/consul/pillar/__init__.py | 65 ++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6caf38b..b4959d9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -275,10 +275,11 @@ class Pillar(object): def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar=None, pillarenv=None): self.minion_id = minion_id + self.ext = ext # Store the file_roots path so we can restore later. Issue 5449 self.actual_file_roots = opts['file_roots'] # use the local file client - self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv) + self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) @@ -317,18 +318,43 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, else: log.error('Pillar data must be a dictionary') - def __valid_ext(self, ext): + def __valid_on_demand_pillar(self, opts): ''' Check to see if the on demand external pillar is allowed ''' - if not isinstance(ext, dict): - return {} - valid = set(('libvirt', 'virtkey')) - if any(key not in valid for key in ext): - return {} - return ext + if not isinstance(self.ext, dict): + log.error( + 'On-demand pillar %s is not formatted as a dictionary', + self.ext + ) + return False + + on_demand_pillar = opts.get('on_demand_pillar', []) + try: + invalid_on_demand_pillar = set([ + x for x in self.ext if x not in on_demand_pillar + ]) + except TypeError: + # Prevent traceback when on_demand_pillar option is malformed + log.error( + 'The on_demand_pillar configuration option is malformed, it ' + 'should be a list of ext_pillar module names' + ) + return False - def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillarenv=None): + if invalid_on_demand_pillar: + log.error( + 'The following ext_pillar modules are not allowed for ' + 'on-demand pillar data: %s. Valid on-demand ext_pillar ' + 'modules are: %s. The valid modules can be adjusted by ' + 'setting the \'on_demand_pillar\' config option.', + ', '.join(sorted(invalid_on_demand_pillar)), + ', '.join(on_demand_pillar), + ) + return False + return True + + def __gen_opts(self, opts_in, grains, saltenv=None, env=None, pillarenv=None): ''' The options need to be altered to conform to the file client ''' @@ -359,11 +385,11 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillaren opts['state_top'] = salt.utils.url.create(opts['state_top'][1:]) else: opts['state_top'] = salt.utils.url.create(opts['state_top']) - if self.__valid_ext(ext): + if self.ext and self.__valid_on_demand_pillar(opts): if 'ext_pillar' in opts: - opts['ext_pillar'].append(ext) + opts['ext_pillar'].append(self.ext) else: - opts['ext_pillar'] = [ext] + opts['ext_pillar'] = [self.ext] return opts def _get_envs(self): @@ -742,6 +768,21 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): ''' if errors is None: errors = [] + try: + # Make sure that on-demand git_pillar is fetched before we try to + # compile the pillar data. git_pillar will fetch a remote when + # the git ext_pillar() func is run, but only for masterless. + if self.ext and 'git' in self.ext \ + and self.opts.get('__role') != 'minion': + # Avoid circular import + import salt.utils.gitfs + from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES + git_pillar = salt.utils.gitfs.GitPillar(self.opts) + git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES) + git_pillar.fetch_remotes() + except TypeError: + # Handle malformed ext_pillar + pass if 'ext_pillar' not in self.opts: return pillar, errors if not isinstance(self.opts['ext_pillar'], list): From 9d724034b0ebda970490d33b61f0a2244542f8a6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 25 Jan 2017 15:08:57 -0600 Subject: [PATCH 495/769] Rename on_demand_pillar to on_demand_ext_pillar --- src/saltext/consul/pillar/__init__.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b4959d9..79adf34 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -318,7 +318,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, else: log.error('Pillar data must be a dictionary') - def __valid_on_demand_pillar(self, opts): + def __valid_on_demand_ext_pillar(self, opts): ''' Check to see if the on demand external pillar is allowed ''' @@ -329,27 +329,25 @@ def __valid_on_demand_pillar(self, opts): ) return False - on_demand_pillar = opts.get('on_demand_pillar', []) + on_demand = opts.get('on_demand_ext_pillar', []) try: - invalid_on_demand_pillar = set([ - x for x in self.ext if x not in on_demand_pillar - ]) + invalid_on_demand = set([x for x in self.ext if x not in on_demand]) except TypeError: - # Prevent traceback when on_demand_pillar option is malformed + # Prevent traceback when on_demand_ext_pillar option is malformed log.error( - 'The on_demand_pillar configuration option is malformed, it ' - 'should be a list of ext_pillar module names' + 'The \'on_demand_ext_pillar\' configuration option is ' + 'malformed, it should be a list of ext_pillar module names' ) return False - if invalid_on_demand_pillar: + if invalid_on_demand: log.error( 'The following ext_pillar modules are not allowed for ' 'on-demand pillar data: %s. Valid on-demand ext_pillar ' 'modules are: %s. The valid modules can be adjusted by ' - 'setting the \'on_demand_pillar\' config option.', - ', '.join(sorted(invalid_on_demand_pillar)), - ', '.join(on_demand_pillar), + 'setting the \'on_demand_ext_pillar\' config option.', + ', '.join(sorted(invalid_on_demand)), + ', '.join(on_demand), ) return False return True @@ -385,7 +383,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, env=None, pillarenv=None): opts['state_top'] = salt.utils.url.create(opts['state_top'][1:]) else: opts['state_top'] = salt.utils.url.create(opts['state_top']) - if self.ext and self.__valid_on_demand_pillar(opts): + if self.ext and self.__valid_on_demand_ext_pillar(opts): if 'ext_pillar' in opts: opts['ext_pillar'].append(self.ext) else: From dff8771a88c75b399f00806310aed86ea70cf77f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sat, 28 Jan 2017 01:01:54 -0600 Subject: [PATCH 496/769] Decrypt pillar data at end of pillar compilation --- src/saltext/consul/pillar/__init__.py | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4ff72b7..5e2f421 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -19,6 +19,7 @@ import salt.transport import salt.utils.url import salt.utils.cache +import salt.utils.crypt from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -873,8 +874,71 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) + decrypt_errors = self.decrypt_pillar(pillar) + if decrypt_errors: + pillar.setdefault('_errors', []).extend(decrypt_errors) return pillar + def decrypt_pillar(self, pillar): + ''' + Decrypt the specified pillar dictionary items, if configured to do so + ''' + errors = [] + if self.opts.get('decrypt_pillar'): + decrypt_pillar = self.opts['decrypt_pillar'] + if not isinstance(decrypt_pillar, dict): + decrypt_pillar = \ + salt.utils.repack_dictlist(self.opts['decrypt_pillar']) + if not decrypt_pillar: + errors.append('decrypt_pillar config option is malformed') + for key, rend in six.iteritems(decrypt_pillar): + ptr = salt.utils.traverse_dict( + pillar, + key, + default=None, + delimiter=self.opts['decrypt_pillar_delimiter']) + if ptr is None: + log.debug('Pillar key %s not present', key) + continue + try: + hash(ptr) + immutable = True + except TypeError: + immutable = False + try: + ret = salt.utils.crypt.decrypt( + ptr, + rend or self.opts['decrypt_pillar_default'], + renderers=self.rend, + opts=self.opts, + valid_rend=self.opts['decrypt_pillar_renderers']) + if immutable: + # Since the key pointed to an immutable type, we need + # to replace it in the pillar dict. First we will find + # the parent, and then we will replace the child key + # with the return data from the renderer. + parent, _, child = key.rpartition( + self.opts['decrypt_pillar_delimiter']) + if not parent: + # key is a top-level key, so the pointer to the + # parent is the pillar dict itself. + ptr = pillar + else: + ptr = salt.utils.traverse_dict( + pillar, + parent, + default=None, + delimiter=self.opts['decrypt_pillar_delimiter']) + if ptr is not None: + ptr[child] = ret + except Exception as exc: + msg = 'Failed to decrypt pillar key \'{0}\': {1}'.format( + key, exc + ) + errors.append(msg) + log.error(msg, exc_info=True) + return errors + # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in # ext_pillar etc. From f7a736e2ae992952fdab7b79023e8bf7b78b7a4e Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 6 Feb 2017 15:38:36 -0700 Subject: [PATCH 497/769] Revert "Cache the renderer in the pillar" This reverts commit 7e89c00fbe19f2dc1c6e328750e85b483b074634. --- src/saltext/consul/pillar/__init__.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5e2f421..4327b3c 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -34,7 +34,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None, rend=None): + pillar=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -50,7 +50,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv, rend=rend) + pillar=pillar, pillarenv=pillarenv) # TODO: migrate everyone to this one! @@ -129,7 +129,7 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None, *args, **kwargs): + pillar=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -269,7 +269,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None, rend=None): + pillar=None, pillarenv=None): self.minion_id = minion_id self.ext = ext if pillarenv is None: @@ -296,10 +296,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.functions = functions self.matcher = salt.minion.Matcher(self.opts, self.functions) - if rend is None: - self.rend = salt.loader.render(self.opts, self.functions) - else: - self.rend = rend + self.rend = salt.loader.render(self.opts, self.functions) ext_pillar_opts = copy.deepcopy(self.opts) # Fix self.opts['file_roots'] so that ext_pillars know the real # location of file_roots. Issue 5951 From 057255e73930ea5cd14fba64ed1b36eb1eafd0ae Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 9 Feb 2017 12:04:11 -0600 Subject: [PATCH 498/769] Allow minion/CLI saltenv/pillarenv to override master when compiling pillar Because we check in the master opts first, if the environment or pillarenv values are set in the master config file, they will take precedence over the values specified in the minion config (or gathered via the CLI). This commit fixes that ordering such that the values passed to __gen_opts() take precedence over the values from the master config. --- src/saltext/consul/pillar/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 79adf34..137756d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -372,11 +372,14 @@ def __gen_opts(self, opts_in, grains, saltenv=None, env=None, pillarenv=None): opts['grains'] = {} else: opts['grains'] = grains - if not opts.get('environment'): - opts['environment'] = saltenv + # Allow minion/CLI saltenv/pillarenv to take precedence over master + opts['environment'] = saltenv \ + if saltenv is not None \ + else opts.get('environment') + opts['pillarenv'] = pillarenv \ + if pillarenv is not None \ + else opts.get('pillarenv') opts['id'] = self.minion_id - if not opts.get('pillarenv'): - opts['pillarenv'] = pillarenv if opts['state_top'].startswith('salt://'): opts['state_top'] = opts['state_top'] elif opts['state_top'].startswith('/'): From 9e00308045d753d18f77bc1f09509d1cbdd4bbf7 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Mon, 20 Feb 2017 18:11:30 -0500 Subject: [PATCH 499/769] prevent billions of inexplicable lines of this: > 17:32 salt.template DEBUG compile template: > 17:32 salt.template ERROR Template does not exist: --- src/saltext/consul/pillar/__init__.py | 29 ++++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 18d9089..ba6e793 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -390,20 +390,21 @@ def get_tops(self): # Gather initial top files try: if self.opts['pillarenv']: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file( + self.opts['state_top'], self.opts['pillarenv']) + + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": From db032433fda807b1460b9d6b3b0bc36a5f1adf23 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 23 Feb 2017 22:06:31 -0600 Subject: [PATCH 500/769] Better logging when an ext_pillar fails to load --- src/saltext/consul/pillar/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7a79652..bf96623 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -10,6 +10,8 @@ import collections import logging import tornado.gen +import sys +import traceback # Import salt libs import salt.loader @@ -817,8 +819,13 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): pillar_dirs, key) except Exception as exc: - errors.append('Failed to load ext_pillar {0}: {1}'.format( - key, exc)) + errors.append( + 'Failed to load ext_pillar {0}: {1}\n{2}'.format( + key, + exc.__str__(), + ''.join(traceback.format_tb(sys.exc_info()[2])), + ) + ) if ext: pillar = merge( pillar, From 225d042b27b4af0d06f1dca38ff1e40819c6e9fa Mon Sep 17 00:00:00 2001 From: dharper Date: Fri, 24 Feb 2017 13:59:52 -0600 Subject: [PATCH 501/769] Checking instance exists in master._get_cached_minion_data when cache.fetch returns None --- src/saltext/consul/cache/__init__.py | 4 ++-- src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index f90e867..e4176cd 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,8 +143,8 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or None if the given - path or key not found. + Return a python object fetched from the cache or and empty dict if + the given path or key not found. :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 0f86c1a..8fee000 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -104,7 +104,7 @@ def fetch(bank, key): try: _, value = api.kv.get(c_key) if value is None: - return value + return {} return __context__['serial'].loads(value['Value']) except Exception as exc: raise SaltCacheError( From 575ee50651f70b7f3a0ad7a4398081d36eb9bec1 Mon Sep 17 00:00:00 2001 From: Dennis Harper Date: Sun, 26 Feb 2017 09:03:26 -0600 Subject: [PATCH 502/769] Update __init__.py --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index e4176cd..542233d 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,7 +143,7 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or and empty dict if + Return a python object fetched from the cache or an empty dict if the given path or key not found. :raises SaltCacheError: From 919d17dc88a4494ab2972939f665b605f18779cd Mon Sep 17 00:00:00 2001 From: Dennis Harper Date: Mon, 27 Feb 2017 10:15:06 -0600 Subject: [PATCH 503/769] Update __init__.py --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 542233d..431f5de 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,7 +143,7 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or an empty dict if + Return a python object fetched from the cache or an empty dict if the given path or key not found. :raises SaltCacheError: From d9de302ad3f370df29f43615aad18f94ed78b79d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Feb 2017 11:46:07 -0600 Subject: [PATCH 504/769] Ensure that ext_pillar begins with pillar_override if ext_pillar_first is True --- src/saltext/consul/pillar/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ba6e793..48f165f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -824,7 +824,9 @@ def compile_pillar(self, ext=True, pillar_dirs=None): top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'], errors = self.ext_pillar({}, pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar( + self.pillar_override, + pillar_dirs) self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) From b542c18c1d161631b6671e7d41e46d061c1bb4b7 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Feb 2017 12:00:22 -0600 Subject: [PATCH 505/769] Remove ext_pillar traceback from returned error db03243 was supposed to affect logging, not the error returned to the CLI. This corrects that oversight. --- src/saltext/consul/pillar/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bf96623..187668a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -820,12 +820,15 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): key) except Exception as exc: errors.append( - 'Failed to load ext_pillar {0}: {1}\n{2}'.format( + 'Failed to load ext_pillar {0}: {1}'.format( key, exc.__str__(), - ''.join(traceback.format_tb(sys.exc_info()[2])), ) ) + log.error( + 'Execption caught loading ext_pillar \'%s\':\n%s', + key, ''.join(traceback.format_tb(sys.exc_info()[2])) + ) if ext: pillar = merge( pillar, From 367ad5f7dd57ed1e0f7c09b52768f0b9b8a2822f Mon Sep 17 00:00:00 2001 From: dharper Date: Fri, 24 Feb 2017 13:59:52 -0600 Subject: [PATCH 506/769] Checking instance exists in master._get_cached_minion_data when cache.fetch returns None --- src/saltext/consul/cache/__init__.py | 4 ++-- src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index f90e867..e4176cd 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,8 +143,8 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or None if the given - path or key not found. + Return a python object fetched from the cache or and empty dict if + the given path or key not found. :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 5dee30b..0615132 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -104,7 +104,7 @@ def fetch(bank, key): try: _, value = api.kv.get(c_key) if value is None: - return value + return {} return __context__['serial'].loads(value['Value']) except Exception as exc: raise SaltCacheError( From 88ebb180f2110fd9de850310491288ccda60c407 Mon Sep 17 00:00:00 2001 From: Dennis Harper Date: Sun, 26 Feb 2017 09:03:26 -0600 Subject: [PATCH 507/769] Update __init__.py --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index e4176cd..542233d 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,7 +143,7 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or and empty dict if + Return a python object fetched from the cache or an empty dict if the given path or key not found. :raises SaltCacheError: From b24b5bf90bc79d9963a7a35829f3dfc17dc47f22 Mon Sep 17 00:00:00 2001 From: Dennis Harper Date: Mon, 27 Feb 2017 10:15:06 -0600 Subject: [PATCH 508/769] Update __init__.py --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 542233d..431f5de 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -143,7 +143,7 @@ def fetch(self, bank, key): added by the driver itself. :return: - Return a python object fetched from the cache or an empty dict if + Return a python object fetched from the cache or an empty dict if the given path or key not found. :raises SaltCacheError: From a7c2410ae18da19cb7f32d02d38f35ff58d1a086 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Mon, 13 Mar 2017 15:30:22 -0500 Subject: [PATCH 509/769] Fix exception in salt-call when master_type is 'disable' If `salt-call` is invoked without using the `--local` flag, it will return an exception when `master_type` is `disable`. Fix this use case. Signed-off-by: Sergey Kizunov --- src/saltext/consul/pillar/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b81d08d..500f211 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -40,10 +40,13 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, ''' Return the correct pillar driver based on the file_client option ''' + file_client = opts['file_client'] + if opts.get('master_type') == 'disable' and file_client == 'remote': + file_client = 'local' ptype = { 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], Pillar) + }.get(file_client, Pillar) # If local pillar and we're caching, run through the cache system first log.debug('Determining pillar cache') if opts['pillar_cache']: @@ -61,10 +64,13 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None ''' Return the correct pillar driver based on the file_client option ''' + file_client = opts['file_client'] + if opts.get('master_type') == 'disable' and file_client == 'remote': + file_client = 'local' ptype = { 'remote': AsyncRemotePillar, 'local': AsyncPillar, - }.get(opts['file_client'], AsyncPillar) + }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) From 12fc0d765a1f60c91195c6ee234066b602bfea10 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Mon, 13 Mar 2017 15:30:22 -0500 Subject: [PATCH 510/769] Fix exception in salt-call when master_type is 'disable' If `salt-call` is invoked without using the `--local` flag, it will return an exception when `master_type` is `disable`. Fix this use case. Signed-off-by: Sergey Kizunov --- src/saltext/consul/pillar/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 48f165f..3eb901e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -36,10 +36,13 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, ''' Return the correct pillar driver based on the file_client option ''' + file_client = opts['file_client'] + if opts.get('master_type') == 'disable' and file_client == 'remote': + file_client = 'local' ptype = { 'remote': RemotePillar, 'local': Pillar - }.get(opts['file_client'], Pillar) + }.get(file_client, Pillar) # If local pillar and we're caching, run through the cache system first log.debug('Determining pillar cache') if opts['pillar_cache']: @@ -57,10 +60,13 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None ''' Return the correct pillar driver based on the file_client option ''' + file_client = opts['file_client'] + if opts.get('master_type') == 'disable' and file_client == 'remote': + file_client = 'local' ptype = { 'remote': AsyncRemotePillar, 'local': AsyncPillar, - }.get(opts['file_client'], AsyncPillar) + }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, pillar=pillar, pillarenv=pillarenv) From 9ed22880806f620c6fed9b7c167b2a67ba9616c7 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Tue, 21 Mar 2017 21:58:02 +0300 Subject: [PATCH 511/769] Bugs and tests fixes. --- src/saltext/consul/cache/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 431f5de..70e3edb 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -9,7 +9,8 @@ from __future__ import absolute_import import time -# Import Salt lobs +# Import Salt libs +import salt.config import salt.loader import salt.syspaths from salt.payload import Serial @@ -54,7 +55,7 @@ def __init__(self, opts, cachedir=None): self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) else: self.cachedir = cachedir - self.driver = opts['cache'] + self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS) self.serial = Serial(opts) self._modules = None From b43939806a083e2aa871d97be3635de29ac4b73b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 24 Mar 2017 17:16:17 +0000 Subject: [PATCH 512/769] System encoding detection compatibility for windows --- src/saltext/consul/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7f32907..495bae6 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -35,10 +35,14 @@ def __define_global_system_encoding_variable__(): # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None - if sys.stdin is not None: + encoding = None + + if not sys.platform.startswith('win') and sys.stdin is not None: + # On linux we can rely on sys.stdin for the encoding since it + # most commonly matches the filesystem encoding. This however + # does not apply to windows encoding = sys.stdin.encoding - else: - encoding = None + if not encoding: # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong From 0465f8f7254df1012d7a5dec9bd302eda4d27e29 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Thu, 19 Jan 2017 15:53:22 -0700 Subject: [PATCH 513/769] Add documentation for the Minion data cache Includes docs for localfs and consul minion data cache modules --- docs/ref/cache/all/salt.cache.consul.rst | 5 +++++ src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/ref/cache/all/salt.cache.consul.rst diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst new file mode 100644 index 0000000..516a3b8 --- /dev/null +++ b/docs/ref/cache/all/salt.cache.consul.rst @@ -0,0 +1,5 @@ +salt.cache.consul module +======================== + +.. automodule:: salt.cache.consul + :members: diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 0615132..8fee000 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -21,7 +21,7 @@ consul.token: None consul.scheme: http consul.consistency: default - consul.dc: None + consul.dc: dc1 consul.verify: True Related docs could be found here: From 52c249e0e02be062b7f92c070b5d5375a55fd1e3 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 24 Mar 2017 14:25:46 -0600 Subject: [PATCH 514/769] Update minion data cache documentation Updats the existing documentation and also adds more documentation to the Multi-Master and Syndic topology files about the addition of the pluggable minion data store. --- src/saltext/consul/cache/consul.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 8fee000..0c84367 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -2,17 +2,23 @@ ''' Minion data cache plugin for Consul key/value data store. +.. versionadded:: 2016.11.2 + It is up to the system administrator to set up and configure the Consul infrastructure. All is needed for this plugin is a working Consul agent -with a read-write access to the key-value storae. +with a read-write access to the key-value store. + +The related documentation can be found in the `Consul documentation`_. + +To enable this cache plugin, the master will need the python client for +Consul installed. This can be easily installed with pip: -The related documentation can be found here: https://www.consul.io/docs/index.html +.. code-block: bash -To enable this cache plugin the master will need the python client for -Consul installed that could be easily done with `pip install python-consul`. + pip install python-consul -Optionally depending on the Consul agent configuration the following values -could be set in the master config, these are the defaults: +Optionally, depending on the Consul agent configuration, the following values +could be set in the master config. These are the defaults: .. code-block:: yaml @@ -24,17 +30,19 @@ consul.dc: dc1 consul.verify: True -Related docs could be found here: -* python-consul: https://python-consul.readthedocs.io/en/latest/#consul +Related docs could be found in the `python-consul documentation`_. -To use the consul as a minion data cache backend set the master `cache` config -value to `consul`: +To use the consul as a minion data cache backend, set the master ``cache`` config +value to ``consul``: .. code-block:: yaml cache: consul -.. versionadded:: 2016.11.2 + +.. _`Consul documentation`: https://www.consul.io/docs/index.html +.. _`python-consul documentation`: https://python-consul.readthedocs.io/en/latest/#consul + ''' from __future__ import absolute_import import logging From d0f5250cdc5ebb90c15e92eb0e159afb9645efbb Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Mon, 27 Mar 2017 16:17:33 -0700 Subject: [PATCH 515/769] ticket 40348: fix pillar include key nested support Extend pillar include support so that nesting is possible for the key directive. Given the following pillar files: foo1.sls: include: - foo: key: two:levels foo2.sls: foon: blah After this commit, it'll result in the following pillar data: two: levels: foon: blah Currently, it results in the following data which is far less useful for usage, and in ability to be addressed: two:levels: foon: blah --- src/saltext/consul/pillar/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 500f211..0c18728 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -688,9 +688,11 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): ) if nstate: if key: - nstate = { - key: nstate - } + # If key is x:y, convert it to {x: {y: nstate}} + for key_fragment in reversed(key.split(":")): + nstate = { + key_fragment: nstate + } state = merge( state, From c578d0f8a4dce10796abe9ab60f328d827c47b4a Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Tue, 28 Mar 2017 19:21:58 +0300 Subject: [PATCH 516/769] In-memory minion data cache. --- src/saltext/consul/cache/__init__.py | 162 ++++++++++++++++++++------- src/saltext/consul/cache/consul.py | 8 +- 2 files changed, 125 insertions(+), 45 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 431f5de..43d3d39 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -7,12 +7,30 @@ # Import Python libs from __future__ import absolute_import +import logging import time # Import Salt lobs +from salt.ext import six +from salt.payload import Serial +from salt.utils.odict import OrderedDict import salt.loader import salt.syspaths -from salt.payload import Serial + +log = logging.getLogger(__name__) + + +def factory(opts, **kwargs): + ''' + Creates and returns the cache class. + If memory caching is enabled by opts MemCache class will be instantiated. + If not Cache class will be returned. + ''' + if opts.get('memcache_expire_seconds', 0): + cls = MemCache + else: + cls = Cache + return cls(opts, **kwargs) class Cache(object): @@ -48,20 +66,25 @@ class Cache(object): Key name is a string identifier of a data container (like a file inside a directory) which will hold the data. ''' - def __init__(self, opts, cachedir=None): + def __init__(self, opts, **kwargs): self.opts = opts - if cachedir is None: - self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) - else: - self.cachedir = cachedir self.driver = opts['cache'] self.serial = Serial(opts) self._modules = None + self._kwargs = kwargs + + def __lazy_init(self): + self._modules = salt.loader.cache(self.opts, self.serial) + fun = '{0}.init_kwargs'.format(self.driver) + if fun in self.modules: + self._kwargs = self.modules[fun](self._kwargs) + else: + self._kwargs = {} @property def modules(self): if self._modules is None: - self._modules = salt.loader.cache(self.opts, self.serial) + self.__lazy_init() return self._modules def cache(self, bank, key, fun, loop_fun=None, **kwargs): @@ -123,11 +146,8 @@ def store(self, bank, key, data): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'store') - try: - return self.modules[fun](bank, key, data, self.cachedir) - except TypeError: - return self.modules[fun](bank, key, data) + fun = '{0}.store'.format(self.driver) + return self.modules[fun](bank, key, data, **self._kwargs) def fetch(self, bank, key): ''' @@ -150,11 +170,8 @@ def fetch(self, bank, key): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'fetch') - try: - return self.modules[fun](bank, key, self.cachedir) - except TypeError: - return self.modules[fun](bank, key) + fun = '{0}.fetch'.format(self.driver) + return self.modules[fun](bank, key, **self._kwargs) def updated(self, bank, key): ''' @@ -177,11 +194,8 @@ def updated(self, bank, key): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'updated') - try: - return self.modules[fun](bank, key, self.cachedir) - except TypeError: - return self.modules[fun](bank, key) + fun = '{0}.updated'.format(self.driver) + return self.modules[fun](bank, key, **self._kwargs) def flush(self, bank, key=None): ''' @@ -201,13 +215,10 @@ def flush(self, bank, key=None): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'flush') - try: - return self.modules[fun](bank, key=key, cachedir=self.cachedir) - except TypeError: - return self.modules[fun](bank, key=key) + fun = '{0}.flush'.format(self.driver) + return self.modules[fun](bank, key=key, **self._kwargs) - def list(self, bank): + def ls(self, bank): ''' Lists entries stored in the specified bank. @@ -223,11 +234,8 @@ def list(self, bank): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'list') - try: - return self.modules[fun](bank, self.cachedir) - except TypeError: - return self.modules[fun](bank) + fun = '{0}.ls'.format(self.driver) + return self.modules[fun](bank, **self._kwargs) def contains(self, bank, key=None): ''' @@ -251,8 +259,86 @@ def contains(self, bank, key=None): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.{1}'.format(self.driver, 'contains') - try: - return self.modules[fun](bank, key, self.cachedir) - except TypeError: - return self.modules[fun](bank, key) + fun = '{0}.contains'.format(self.driver) + return self.modules[fun](bank, key, **self._kwargs) + + +class MemCache(Cache): + ''' + Short-lived in-memory cache store keeping values on time and/or size (count) + basis. + ''' + # {: odict({: [atime, data], ...}), ...} + data = {} + + def __init__(self, opts, **kwargs): + super(MemCache, self).__init__(opts, **kwargs) + self.expire = opts.get('memcache_expire_seconds', 10) + self.max = opts.get('memcache_max_items', 1024) + self.cleanup = opts.get('memcache_full_cleanup', False) + self.debug = opts.get('memcache_debug', False) + if self.debug: + self.call = 0 + self.hit = 0 + self._storage = None + + @classmethod + def __cleanup(cls, expire): + now = time.time() + for storage in six.itervalues(cls.data): + for key, data in list(storage.items()): + if data[0] + expire < now: + del storage[key] + + def _get_storage_id(self): + fun = '{0}.storage_id'.format(self.driver) + if fun in self.modules: + return self.modules[fun](self.kwargs) + else: + return self.driver + + @property + def storage(self): + if self._storage is None: + storage_id = self._get_storage_id() + if storage_id not in MemCache.data: + MemCache.data[storage_id] = OrderedDict() + self._storage = MemCache.data[storage_id] + return self._storage + + def fetch(self, bank, key): + if self.debug: + self.call += 1 + now = time.time() + record = self.storage.pop((bank, key), None) + # Have a cached value for the key + if record is not None and record[0] + self.expire >= now: + if self.debug: + self.hit += 1 + log.trace('MemCache stats (call/hit/rate): ' + '{0}/{1}/{2}'.format(self.call, + self.hit, + float(self.hit) / self.call)) + # update atime and return + record[0] = now + self.storage[(bank, key)] = record + return record[1] + + # Have no value for the key or value is expired + data = super(MemCache, self).fetch(bank, key) + self.storage[(bank, key)] = [now, data] + return data + + def store(self, bank, key, data): + self.storage.pop((bank, key), None) + super(MemCache, self).store(bank, key, data) + if len(self.storage) >= self.max: + if self.cleanup: + MemCache.__cleanup(self.expire) + else: + self.storage.popitem(last=False) + self.storage[(bank, key)] = [time.time(), data] + + def flush(self, bank, key=None): + self.storage.pop((bank, key), None) + super(MemCache, self).flush(bank, key) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 0c84367..a121e7e 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -54,9 +54,6 @@ from salt.exceptions import SaltCacheError -# Don't shadow built-ins -__func_alias__ = {'list_': 'list'} - log = logging.getLogger(__name__) api = None @@ -140,7 +137,7 @@ def flush(bank, key=None): ) -def list_(bank): +def ls(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' @@ -164,9 +161,6 @@ def list_(bank): return keys -getlist = list_ - - def contains(bank, key): ''' Checks if the specified bank contains the specified key. From e313bc3aa27ea2bd70b56ff42f5ce9269bebca87 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 29 Mar 2017 01:04:04 +0100 Subject: [PATCH 517/769] Take into account the comment from @damon-atkins. Improved MS Win support. --- src/saltext/consul/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 495bae6..84e5d84 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -61,7 +61,18 @@ def __define_global_system_encoding_variable__(): # This is most likely ascii which is not the best but we were # unable to find a better encoding. If this fails, we fall all # the way back to ascii - encoding = sys.getdefaultencoding() or 'ascii' + encoding = sys.getdefaultencoding() + if not encoding: + if sys.platform.startswith('darwin'): + # Mac OS X uses UTF-8 + encoding = 'utf-8' + elif sys.platform.startswith('win'): + # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” + # to refer to whatever the currently configured encoding is. + encoding = 'mbcs' + else: + # On linux default to ascii as a last resort + encoding = 'ascii' # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: From 0588b59f056f52727f575e371e1a2565710155b6 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 3 Apr 2017 19:48:04 +0300 Subject: [PATCH 518/769] Memcache documentation and minor updates. --- src/saltext/consul/cache/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 43d3d39..ada7f52 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -289,6 +289,8 @@ def __cleanup(cls, expire): for key, data in list(storage.items()): if data[0] + expire < now: del storage[key] + else: + break def _get_storage_id(self): fun = '{0}.storage_id'.format(self.driver) @@ -315,7 +317,7 @@ def fetch(self, bank, key): if record is not None and record[0] + self.expire >= now: if self.debug: self.hit += 1 - log.trace('MemCache stats (call/hit/rate): ' + log.debug('MemCache stats (call/hit/rate): ' '{0}/{1}/{2}'.format(self.call, self.hit, float(self.hit) / self.call)) @@ -326,6 +328,11 @@ def fetch(self, bank, key): # Have no value for the key or value is expired data = super(MemCache, self).fetch(bank, key) + if len(self.storage) >= self.max: + if self.cleanup: + MemCache.__cleanup(self.expire) + if len(self.storage) >= self.max: + self.storage.popitem(last=False) self.storage[(bank, key)] = [now, data] return data @@ -335,7 +342,7 @@ def store(self, bank, key, data): if len(self.storage) >= self.max: if self.cleanup: MemCache.__cleanup(self.expire) - else: + if len(self.storage) >= self.max: self.storage.popitem(last=False) self.storage[(bank, key)] = [time.time(), data] From 4821fe3bbc48b741789563395bd70c3998610d61 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 3 Apr 2017 11:02:08 -0600 Subject: [PATCH 519/769] Add cachedir variable setting back in after bad merge --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index c16d1ca..558d519 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -67,7 +67,7 @@ class Cache(object): Key name is a string identifier of a data container (like a file inside a directory) which will hold the data. ''' - def __init__(self, opts, **kwargs): + def __init__(self, opts, cachedir=None, **kwargs): self.opts = opts if cachedir is None: self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) From 99699953e273da65609037bd6821c7c19e572660 Mon Sep 17 00:00:00 2001 From: Joseph Hall Date: Fri, 31 Mar 2017 14:40:48 -0600 Subject: [PATCH 520/769] Add __func_alias__ back in --- src/saltext/consul/cache/consul.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index a121e7e..d7cbb1f 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -61,6 +61,8 @@ # Define the module's virtual name __virtualname__ = 'consul' +__func_alias__ = {'ls': 'list'} + def __virtual__(): ''' From af0687c249e23fb5a16ea07393be35e2265ecc10 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Wed, 5 Apr 2017 16:45:32 +0300 Subject: [PATCH 521/769] A quick fix for Cache has no 'list' attribute. --- src/saltext/consul/cache/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index ada7f52..bf97112 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -237,6 +237,8 @@ def ls(self, bank): fun = '{0}.ls'.format(self.driver) return self.modules[fun](bank, **self._kwargs) + list = ls + def contains(self, bank, key=None): ''' Checks if the specified bank contains the specified key. From d4beb03c63a1875dff462c4da622b06f7ced33b1 Mon Sep 17 00:00:00 2001 From: Harris Dimitriou Date: Thu, 25 Aug 2016 20:30:04 +0300 Subject: [PATCH 522/769] fixed 'Template not found' error by adding a control in get_tops() not to look for top file if pillarenv is defined and is not equal to file_roots --- src/saltext/consul/pillar/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0c18728..70ca76d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -411,13 +411,21 @@ def get_tops(self): # Gather initial top files try: if self.opts['pillarenv']: - top = self.client.cache_file( - self.opts['state_top'], self.opts['pillarenv']) - - if top: + # If the specified pillarenv is not present in the available + # pillar environments, do not cache the pillar top file. + if self.opts['pillarenv'] not in self.opts['file_roots']: + log.debug( + 'pillarenv \'%s\' not found in the configured pillar ' + 'environments (%s)', + self.opts['pillarenv'], ', '.join(self.opts('file_roots')) + ) + else: tops[self.opts['pillarenv']] = [ compile_template( - top, + self.client.cache_file( + self.opts['state_top'], + self.opts['pillarenv'] + ), self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], From 48ebaa361a69c2c277646c9b59d77ccc72d843c7 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 11 Apr 2017 11:26:46 -0500 Subject: [PATCH 523/769] Fix regression in merge conflict resolution See https://github.com/saltstack/salt/commit/d4beb03c63a1875dff462c4da622b06f7ced33b1#commitcomment-21715967 --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 70ca76d..6b1c2b9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -417,7 +417,7 @@ def get_tops(self): log.debug( 'pillarenv \'%s\' not found in the configured pillar ' 'environments (%s)', - self.opts['pillarenv'], ', '.join(self.opts('file_roots')) + self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: tops[self.opts['pillarenv']] = [ From 31d4b3d1adcccf72870e731de3a02fabe3c82142 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 18 Apr 2017 13:41:07 -0600 Subject: [PATCH 524/769] Use salt.utils.yamlloader with SaltYamlSafeLoader as the Loader with yaml.load --- src/saltext/consul/pillar/consul_pillar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 0a4384c..3ca86ea 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -126,10 +126,11 @@ import logging import re import yaml -import salt.utils.minions from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge +import salt.utils.minions +from salt.utils.yamlloader import SaltYamlSafeLoader # Import third party libs try: @@ -251,7 +252,10 @@ def pillar_format(ret, keys, value): # If value is not None then it's a string # Use YAML to parse the data # YAML strips whitespaces unless they're surrounded by quotes - pillar_value = yaml.load(value) + pillar_value = yaml.load( + value, + Loader=SaltYamlSafeLoader + ) keyvalue = keys.pop() pil = {keyvalue: pillar_value} From c5045b85932bb34e30280f261404de73df9eb3ea Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Wed, 19 Apr 2017 10:44:18 +0800 Subject: [PATCH 525/769] Fix 'root=/...' references in consul_pillar documentation: 'keys should not start with a forward slash'! --- src/saltext/consul/pillar/consul_pillar.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 0a4384c..7fde2d2 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -72,7 +72,7 @@ - consul: my_consul_config ext_pillar: - - consul: my_consul_config root=/salt + - consul: my_consul_config root=salt Using these configuration profiles, multiple consul sources may also be used: @@ -88,9 +88,9 @@ .. code-block:: yaml ext_pillar: - - consul: my_consul_config root=/salt/%(minion_id)s - - consul: my_consul_config root=/salt/%(role)s - - consul: my_consul_config root=/salt/%(environment)s + - consul: my_consul_config root=salt/%(minion_id)s + - consul: my_consul_config root=salt/%(role)s + - consul: my_consul_config root=salt/%(environment)s Minion-specific values may override shared values when the minion-specific root appears after the shared root: @@ -98,8 +98,8 @@ .. code-block:: yaml ext_pillar: - - consul: my_consul_config root=/salt-shared - - consul: my_other_consul_config root=/salt-private/%(minion_id)s + - consul: my_consul_config root=salt-shared + - consul: my_other_consul_config root=salt-private/%(minion_id)s If using the ``role`` or ``environment`` grain in the consul key path, be sure to define it using `/etc/salt/grains`, or similar: From d36d0ca7300a9c71b5183c9f469b4973aacaf993 Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Wed, 19 Apr 2017 10:44:18 +0800 Subject: [PATCH 526/769] Fix 'root=/...' references in consul_pillar documentation: 'keys should not start with a forward slash'! --- src/saltext/consul/pillar/consul_pillar.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 0a4384c..7fde2d2 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -72,7 +72,7 @@ - consul: my_consul_config ext_pillar: - - consul: my_consul_config root=/salt + - consul: my_consul_config root=salt Using these configuration profiles, multiple consul sources may also be used: @@ -88,9 +88,9 @@ .. code-block:: yaml ext_pillar: - - consul: my_consul_config root=/salt/%(minion_id)s - - consul: my_consul_config root=/salt/%(role)s - - consul: my_consul_config root=/salt/%(environment)s + - consul: my_consul_config root=salt/%(minion_id)s + - consul: my_consul_config root=salt/%(role)s + - consul: my_consul_config root=salt/%(environment)s Minion-specific values may override shared values when the minion-specific root appears after the shared root: @@ -98,8 +98,8 @@ .. code-block:: yaml ext_pillar: - - consul: my_consul_config root=/salt-shared - - consul: my_other_consul_config root=/salt-private/%(minion_id)s + - consul: my_consul_config root=salt-shared + - consul: my_other_consul_config root=salt-private/%(minion_id)s If using the ``role`` or ``environment`` grain in the consul key path, be sure to define it using `/etc/salt/grains`, or similar: From d877fccaf05c7eeb5c4ff6b39ba7ce97e3f78f27 Mon Sep 17 00:00:00 2001 From: Daniel Santos Date: Fri, 28 Apr 2017 10:13:56 -0600 Subject: [PATCH 527/769] Fix consul module "AttributeError: 'dict' object has no attribute 'json'" Consul API has changed a little bit, some small adjustments to make the module work again --- src/saltext/consul/modules/consul.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 73b661c..4ed96ef 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -93,7 +93,6 @@ def _query(function, ret['data'] = 'Key not found.' ret['res'] = False else: - result = result.json() if result: ret['data'] = result ret['res'] = True @@ -558,7 +557,7 @@ def agent_maintenance(consul_url=None, **kwargs): function = 'agent/maintenance' res = _query(consul_url=consul_url, function=function, - method='GET', + method='PUT', query_params=query_params) if res['res']: ret['res'] = True From 3a94c38712cad74ab30de5465c42119902477ddf Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 16 May 2017 15:40:41 -0700 Subject: [PATCH 528/769] Swapping the order in the func_alias so the ls function is available. --- src/saltext/consul/cache/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index d7cbb1f..b545c96 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -61,7 +61,7 @@ # Define the module's virtual name __virtualname__ = 'consul' -__func_alias__ = {'ls': 'list'} +__func_alias__ = {'list': 'ls'} def __virtual__(): From 551bbff904b45ee13aef270903a0ace85544a9ac Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Thu, 25 May 2017 15:18:27 +0800 Subject: [PATCH 529/769] pillar: target's state list support wildcard in top.sls --- src/saltext/consul/pillar/__init__.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6b1c2b9..cee40d5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -6,6 +6,7 @@ # Import python libs from __future__ import absolute_import import copy +import fnmatch import os import collections import logging @@ -289,6 +290,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) + self.avail = self.__gather_avail() if opts.get('file_client', '') == 'local': opts['grains'] = grains @@ -359,6 +361,15 @@ def __valid_on_demand_ext_pillar(self, opts): return False return True + def __gather_avail(self): + ''' + Gather the lists of available sls data from the master + ''' + avail = {} + for saltenv in self._get_envs(): + avail[saltenv] = self.client.list_states(saltenv) + return avail + def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): ''' The options need to be altered to conform to the file client @@ -722,8 +733,20 @@ def render_pillar(self, matches, errors=None): if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): + pstatesfile = [] mods = set() - for sls in pstates: + for sls_match in pstates: + try: + pstatefiles = fnmatch.filter(self.avail[saltenv], sls_match) + except KeyError: + errors.extend( + ['No matching pillar environment for environment ' + '\'{0}\' found'.format(saltenv)] + ) + if not pstatefiles: + pstatefiles = [sls_match] + + for sls in pstatefiles: pstate, mods, err = self.render_pstate(sls, saltenv, mods) if err: From 82103b63a9ce6b86817f4037f56d5f6ce3bb61a8 Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Fri, 26 May 2017 10:06:06 +0800 Subject: [PATCH 530/769] fix integration test --- src/saltext/consul/pillar/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index cee40d5..f27eaa7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -733,18 +733,21 @@ def render_pillar(self, matches, errors=None): if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): - pstatesfile = [] + pstatefiles = [] mods = set() for sls_match in pstates: + matched_pstates = [] try: - pstatefiles = fnmatch.filter(self.avail[saltenv], sls_match) + matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match) except KeyError: errors.extend( ['No matching pillar environment for environment ' '\'{0}\' found'.format(saltenv)] ) - if not pstatefiles: - pstatefiles = [sls_match] + if matched_pstates: + pstatefiles.extend(matched_pstates) + else: + pstatefiles.append(sls_match) for sls in pstatefiles: pstate, mods, err = self.render_pstate(sls, saltenv, mods) From e84da354731f7e5f342cf084c991f4682f8f63f1 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 31 May 2017 11:43:38 -0600 Subject: [PATCH 531/769] Set default for consul_pillar to None If these do not default to None, they will default to an empty string, which could cause the pillar tree to leak to minions it should't. Also, allow role and environment to be pulled from pillars or minion config by using config.get Fixes #41478 --- src/saltext/consul/pillar/consul_pillar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 17042c6..0d10b80 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -189,8 +189,8 @@ def ext_pillar(minion_id, client = get_conn(__opts__, opts['profile']) - role = __salt__['grains.get']('role') - environment = __salt__['grains.get']('environment') + role = __salt__['grains.get']('role', None) + environment = __salt__['grains.get']('environment', None) # put the minion's ID in the path if necessary opts['root'] %= { 'minion_id': minion_id, From bca57e11b62e58600fc71a54601d13859246d6f9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 9 Jun 2017 11:34:59 -0500 Subject: [PATCH 532/769] salt.pillar: rename the "pillar" argument to "pillar_override" This differentiates CLI pillar override with an eventual future "initial_pillar" argument which will seed the Pillar class with initial data. --- src/saltext/consul/pillar/__init__.py | 72 ++++++++++++--------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3eb901e..a2d1dcb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -32,7 +32,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -49,14 +49,14 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, log.info('Compiling pillar from cache') log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -68,7 +68,7 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None 'local': AsyncPillar, }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) class AsyncRemotePillar(object): @@ -76,7 +76,7 @@ class AsyncRemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -85,12 +85,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.channel = salt.transport.client.AsyncReqChannel.factory(opts) if pillarenv is not None or 'pillarenv' not in self.opts: self.opts['pillarenv'] = pillarenv - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') @tornado.gen.coroutine def compile_pillar(self): @@ -129,7 +127,7 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -138,12 +136,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.channel = salt.transport.Channel.factory(opts) if pillarenv is not None or 'pillarenv' not in self.opts: self.opts['pillarenv'] = pillarenv - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') def compile_pillar(self): ''' @@ -175,8 +171,8 @@ class PillarCache(object): ''' Return a cached pillar if it exists, otherwise cache it. - Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs. - Each saltenv contains a pillar dict + Pillar caches are structed in two diminensions: minion_id with a dict of + saltenvs. Each saltenv contains a pillar dict Example data structure: @@ -187,7 +183,7 @@ class PillarCache(object): ''' # TODO ABC? def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -197,7 +193,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.minion_id = minion_id self.ext = ext self.functions = functions - self.pillar = pillar + self.pillar_override = pillar_override self.pillarenv = pillarenv if saltenv is None: @@ -226,13 +222,13 @@ def fetch_pillar(self): ''' log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) fresh_pillar = Pillar(self.opts, - self.grains, - self.minion_id, - self.saltenv, - ext=self.ext, - functions=self.functions, - pillar=self.pillar, - pillarenv=self.pillarenv) + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar_override=self.pillar_override, + pillarenv=self.pillarenv) return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs @@ -265,7 +261,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.minion_id = minion_id self.ext = ext # Store the file_roots path so we can restore later. Issue 5449 @@ -303,12 +299,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') def __valid_on_demand_ext_pillar(self, opts): ''' @@ -784,7 +778,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): return pillar, errors ext = None # Bring in CLI pillar data - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, @@ -863,7 +857,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, From dc8a5f7f2a36a309887db8acd5f993ae92f08bcc Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 21 Jun 2017 14:51:02 -0600 Subject: [PATCH 533/769] Merge forward from 2016.11 to 2017 --- src/saltext/consul/pillar/__init__.py | 66 ++++++++++++++++----------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6b1c2b9..cb008b4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -36,7 +36,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -53,14 +53,14 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, log.info('Compiling pillar from cache') log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -72,7 +72,7 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None 'local': AsyncPillar, }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) class AsyncRemotePillar(object): @@ -80,7 +80,7 @@ class AsyncRemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -89,6 +89,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.channel = salt.transport.client.AsyncReqChannel.factory(opts) if pillarenv is not None: self.opts['pillarenv'] = pillarenv +<<<<<<< HEAD elif self.opts.get('pillarenv_from_saltenv', False): self.opts['pillarenv'] = saltenv elif 'pillarenv' not in self.opts: @@ -99,6 +100,12 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.pillar_override = pillar else: log.error('Pillar data must be a dictionary') +======= + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') +>>>>>>> 2016.11 @tornado.gen.coroutine def compile_pillar(self): @@ -137,7 +144,7 @@ class RemotePillar(object): Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -146,6 +153,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.channel = salt.transport.Channel.factory(opts) if pillarenv is not None: self.opts['pillarenv'] = pillarenv +<<<<<<< HEAD elif self.opts.get('pillarenv_from_saltenv', False): self.opts['pillarenv'] = saltenv elif 'pillarenv' not in self.opts: @@ -156,6 +164,12 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.pillar_override = pillar else: log.error('Pillar data must be a dictionary') +======= + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') +>>>>>>> 2016.11 def compile_pillar(self): ''' @@ -187,8 +201,8 @@ class PillarCache(object): ''' Return a cached pillar if it exists, otherwise cache it. - Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs. - Each saltenv contains a pillar dict + Pillar caches are structed in two diminensions: minion_id with a dict of + saltenvs. Each saltenv contains a pillar dict Example data structure: @@ -199,7 +213,7 @@ class PillarCache(object): ''' # TODO ABC? def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -209,7 +223,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.minion_id = minion_id self.ext = ext self.functions = functions - self.pillar = pillar + self.pillar_override = pillar_override self.pillarenv = pillarenv if saltenv is None: @@ -238,13 +252,13 @@ def fetch_pillar(self): ''' log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) fresh_pillar = Pillar(self.opts, - self.grains, - self.minion_id, - self.saltenv, - ext=self.ext, - functions=self.functions, - pillar=self.pillar, - pillarenv=self.pillarenv) + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar_override=self.pillar_override, + pillarenv=self.pillarenv) return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs @@ -277,7 +291,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None): self.minion_id = minion_id self.ext = ext if pillarenv is None: @@ -318,12 +332,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') def __valid_on_demand_ext_pillar(self, opts): ''' @@ -809,7 +821,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): return pillar, errors ext = None # Bring in CLI pillar data - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, @@ -896,7 +908,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, From 879e793c31322f9d575c53fa0ea58496e25cbb94 Mon Sep 17 00:00:00 2001 From: Phil Porada Date: Fri, 30 Jun 2017 15:31:36 -0400 Subject: [PATCH 534/769] Fixes grammar --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 814027d..a954ba0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -568,7 +568,7 @@ def get_top(self): merged_tops = self.merge_tops(tops) except TypeError as err: merged_tops = OrderedDict() - errors.append('Error encountered while render pillar top file.') + errors.append('Error encountered while rendering pillar top file.') return merged_tops, errors def top_matches(self, top): From d018a70eb40b4e4a1a4e3f86957d89596a34ba86 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 3 Jul 2017 09:55:19 -0600 Subject: [PATCH 535/769] pass cachedir from kwargs in cache --- src/saltext/consul/cache/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index b8f9110..4c66dd3 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -77,6 +77,7 @@ def __init__(self, opts, cachedir=None, **kwargs): self.serial = Serial(opts) self._modules = None self._kwargs = kwargs + self._kwargs['cachedir'] = self.cachedir def __lazy_init(self): self._modules = salt.loader.cache(self.opts, self.serial) From 5c0e25e2bf7dda0daa0d5d94afb047f1fc641b47 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 5 Jul 2017 14:19:30 -0500 Subject: [PATCH 536/769] Revert "decode pillar on minion side on receiving" This reverts commit bdc3a997322b9eebee602125867244b9c40f4cdb. --- src/saltext/consul/pillar/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bdda47f..a62e11d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -27,7 +27,6 @@ from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict from salt.version import __version__ -from salt.utils.locales import decode_recursively # Import 3rd-party libs import salt.ext.six as six @@ -168,7 +167,7 @@ def compile_pillar(self): '{1}'.format(type(ret_pillar).__name__, ret_pillar) ) return {} - return decode_recursively(ret_pillar) + return ret_pillar class PillarCache(object): From 9e428ea665d02b25d4bed526bce586c5f343c49a Mon Sep 17 00:00:00 2001 From: Andrew Pashkin Date: Sat, 15 Oct 2016 00:59:15 +0300 Subject: [PATCH 537/769] Factor out pillar merging in salt.pillar.Pillar into a separate helper method --- src/saltext/consul/pillar/__init__.py | 54 +++++++++++---------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3af3dce..476afbf 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -396,6 +396,21 @@ def _get_envs(self): envs.update(list(self.opts['file_roots'])) return envs + def __merge(self, *items): + ''' + Merge 'items' according to Pillar's merge strategy and other options. + ''' + return six.moves.reduce( + lambda a, b: merge( + a, + b, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False) + ), + items + ) + def get_tops(self): ''' Gather the top files @@ -698,12 +713,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key_fragment: nstate } - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + state = self.__merge(state, nstate) if err: errors += err @@ -754,12 +764,7 @@ def render_pillar(self, matches, errors=None): ) ) continue - pillar = merge( - pillar, - pstate, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = self.__merge(pillar, pstate) return pillar, errors @@ -821,11 +826,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): ext = None # Bring in CLI pillar data if self.pillar_override: - pillar = merge(pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = self.__merge(pillar, self.pillar_override) for run in self.opts['ext_pillar']: if not isinstance(run, dict): @@ -858,12 +859,7 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): key, ''.join(traceback.format_tb(sys.exc_info()[2])) ) if ext: - pillar = merge( - pillar, - ext, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = self.__merge(pillar, ext) ext = None return pillar, errors @@ -880,11 +876,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) - pillar = merge(self.opts['pillar'], - pillar, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = self.__merge(self.opts['pillar'], pillar) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -908,11 +900,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): pillar['_errors'] = errors if self.pillar_override: - pillar = merge(pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = self.__merge(pillar, self.pillar_override) decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: From ab8595da0dd78733fffb3594728f0306e27220fc Mon Sep 17 00:00:00 2001 From: Andrew Pashkin Date: Fri, 14 Oct 2016 16:28:15 +0300 Subject: [PATCH 538/769] Make possible to reference pillars, that come before current pillar in the top file --- src/saltext/consul/pillar/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 476afbf..f284be5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -613,10 +613,12 @@ def top_matches(self, top): env_matches.append(item) return matches - def render_pstate(self, sls, saltenv, mods, defaults=None): + def render_pstate(self, sls, saltenv, mods, renderers=None, defaults=None): ''' Collect a single pillar sls file and render it ''' + if renderers is None: + renderers = self.rend if defaults is None: defaults = {} err = '' @@ -656,7 +658,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state = None try: state = compile_template(fn_, - self.rend, + renderers, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], @@ -724,7 +726,10 @@ def render_pillar(self, matches, errors=None): Extract the sls pillar files from the matches and render them into the pillar ''' - pillar = copy.copy(self.pillar_override) + pillar = self.__merge( + self.opts.get('pillar', {}), + self.pillar_override + ) if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): @@ -745,7 +750,16 @@ def render_pillar(self, matches, errors=None): pstatefiles.append(sls_match) for sls in pstatefiles: - pstate, mods, err = self.render_pstate(sls, saltenv, mods) + opts = dict(self.opts, pillar=pillar) + utils = salt.loader.utils(opts) + functions = salt.loader.minion_mods(opts, utils=utils) + renderers = salt.loader.render(opts, functions) + pstate, mods, err = self.render_pstate( + sls, + saltenv, + mods, + renderers=renderers + ) if err: errors += err From 8de57e85be289976602bab3ef7504d5cb5cdbd8d Mon Sep 17 00:00:00 2001 From: Andrew Pashkin Date: Wed, 19 Jul 2017 12:25:22 +0300 Subject: [PATCH 539/769] Revert "Make possible to reference previous pillars from subsequent pillars, as they specified in the top file" --- src/saltext/consul/pillar/__init__.py | 76 +++++++++++++-------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f284be5..3af3dce 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -396,21 +396,6 @@ def _get_envs(self): envs.update(list(self.opts['file_roots'])) return envs - def __merge(self, *items): - ''' - Merge 'items' according to Pillar's merge strategy and other options. - ''' - return six.moves.reduce( - lambda a, b: merge( - a, - b, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False) - ), - items - ) - def get_tops(self): ''' Gather the top files @@ -613,12 +598,10 @@ def top_matches(self, top): env_matches.append(item) return matches - def render_pstate(self, sls, saltenv, mods, renderers=None, defaults=None): + def render_pstate(self, sls, saltenv, mods, defaults=None): ''' Collect a single pillar sls file and render it ''' - if renderers is None: - renderers = self.rend if defaults is None: defaults = {} err = '' @@ -658,7 +641,7 @@ def render_pstate(self, sls, saltenv, mods, renderers=None, defaults=None): state = None try: state = compile_template(fn_, - renderers, + self.rend, self.opts['renderer'], self.opts['renderer_blacklist'], self.opts['renderer_whitelist'], @@ -715,7 +698,12 @@ def render_pstate(self, sls, saltenv, mods, renderers=None, defaults=None): key_fragment: nstate } - state = self.__merge(state, nstate) + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) if err: errors += err @@ -726,10 +714,7 @@ def render_pillar(self, matches, errors=None): Extract the sls pillar files from the matches and render them into the pillar ''' - pillar = self.__merge( - self.opts.get('pillar', {}), - self.pillar_override - ) + pillar = copy.copy(self.pillar_override) if errors is None: errors = [] for saltenv, pstates in six.iteritems(matches): @@ -750,16 +735,7 @@ def render_pillar(self, matches, errors=None): pstatefiles.append(sls_match) for sls in pstatefiles: - opts = dict(self.opts, pillar=pillar) - utils = salt.loader.utils(opts) - functions = salt.loader.minion_mods(opts, utils=utils) - renderers = salt.loader.render(opts, functions) - pstate, mods, err = self.render_pstate( - sls, - saltenv, - mods, - renderers=renderers - ) + pstate, mods, err = self.render_pstate(sls, saltenv, mods) if err: errors += err @@ -778,7 +754,12 @@ def render_pillar(self, matches, errors=None): ) ) continue - pillar = self.__merge(pillar, pstate) + pillar = merge( + pillar, + pstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) return pillar, errors @@ -840,7 +821,11 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): ext = None # Bring in CLI pillar data if self.pillar_override: - pillar = self.__merge(pillar, self.pillar_override) + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) for run in self.opts['ext_pillar']: if not isinstance(run, dict): @@ -873,7 +858,12 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): key, ''.join(traceback.format_tb(sys.exc_info()[2])) ) if ext: - pillar = self.__merge(pillar, ext) + pillar = merge( + pillar, + ext, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) ext = None return pillar, errors @@ -890,7 +880,11 @@ def compile_pillar(self, ext=True, pillar_dirs=None): self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) - pillar = self.__merge(self.opts['pillar'], pillar) + pillar = merge(self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -914,7 +908,11 @@ def compile_pillar(self, ext=True, pillar_dirs=None): pillar['_errors'] = errors if self.pillar_override: - pillar = self.__merge(pillar, self.pillar_override) + pillar = merge(pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: From afdab970a2b9d7ec7d02d1d22086146984bde9af Mon Sep 17 00:00:00 2001 From: remijouannet Date: Wed, 26 Jul 2017 01:27:38 +0200 Subject: [PATCH 540/769] update consul module following this documentation https://www.consul.io/api/acl.html --- src/saltext/consul/modules/consul.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 049831b..33d746c 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1978,6 +1978,8 @@ def acl_create(consul_url=None, **kwargs): Create a new ACL token. :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to create + leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2008,6 +2010,9 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') + if 'id' in kwargs: + data['ID'] = kwargs['id'] + if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2126,7 +2131,7 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) + function = 'acl/destroy/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, data=data, method='PUT', From d23ebb7d5d059d3a2e790da1ed39b57fc23499f7 Mon Sep 17 00:00:00 2001 From: remijouannet Date: Wed, 26 Jul 2017 01:27:38 +0200 Subject: [PATCH 541/769] update consul module following this documentation https://www.consul.io/api/acl.html --- src/saltext/consul/modules/consul.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 4ed96ef..2629971 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1972,6 +1972,8 @@ def acl_create(consul_url=None, **kwargs): Create a new ACL token. :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to create + leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2002,6 +2004,9 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') + if 'id' in kwargs: + data['ID'] = kwargs['id'] + if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2120,7 +2125,7 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) + function = 'acl/destroy/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, data=data, method='PUT', From a1d5e2b6cb2f93372a83cac13df6b7493d549754 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 24 Jul 2017 20:47:15 -0500 Subject: [PATCH 542/769] Use explicit unicode strings + break up salt.utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR is part of what will be an ongoing effort to use explicit unicode strings in Salt. Because Python 3 does not suport Python 2's raw unicode string syntax (i.e. `ur'\d+'`), we must use `salt.utils.locales.sdecode()` to ensure that the raw string is unicode. However, because of how `salt/utils/__init__.py` has evolved into the hulking monstrosity it is today, this means importing a large module in places where it is not needed, which could negatively impact performance. For this reason, this PR also breaks out some of the functions from `salt/utils/__init__.py` into new/existing modules under `salt/utils/`. The long term goal will be that the modules within this directory do not depend on importing `salt.utils`. A summary of the changes in this PR is as follows: * Moves the following functions from `salt.utils` to new locations (including a deprecation warning if invoked from `salt.utils`): `to_bytes`, `to_str`, `to_unicode`, `str_to_num`, `is_quoted`, `dequote`, `is_hex`, `is_bin_str`, `rand_string`, `contains_whitespace`, `clean_kwargs`, `invalid_kwargs`, `which`, `which_bin`, `path_join`, `shlex_split`, `rand_str`, `is_windows`, `is_proxy`, `is_linux`, `is_darwin`, `is_sunos`, `is_smartos`, `is_smartos_globalzone`, `is_smartos_zone`, `is_freebsd`, `is_netbsd`, `is_openbsd`, `is_aix` * Moves the functions already deprecated by @rallytime to the bottom of `salt/utils/__init__.py` for better organization, so we can keep the deprecated ones separate from the ones yet to be deprecated as we continue to break up `salt.utils` * Updates `salt/*.py` and all files under `salt/client/` to use explicit unicode string literals. * Gets rid of implicit imports of `salt.utils` (e.g. `from salt.utils import foo` becomes `import salt.utils.foo as foo`). * Renames the `test.rand_str` function to `test.random_hash` to more accurately reflect what it does * Modifies `salt.utils.stringutils.random()` (née `salt.utils.rand_string()`) such that it returns a string matching the passed size. Previously this function would get `size` bytes from `os.urandom()`, base64-encode it, and return the result, which would in most cases not be equal to the passed size. --- src/saltext/consul/__init__.py | 24 +++++++++++++----------- src/saltext/consul/pillar/__init__.py | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 84e5d84..21302c9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import warnings +# future lint: disable=non-unicode-string # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( 'once', # Show once @@ -14,18 +15,19 @@ DeprecationWarning, # This filter is for DeprecationWarnings r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' ) +# future lint: enable=non-unicode-string # While we are supporting Python2.6, hide nested with-statements warnings warnings.filterwarnings( - 'ignore', - 'With-statements now directly support multiple context managers', + u'ignore', + u'With-statements now directly support multiple context managers', DeprecationWarning ) # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( - 'ignore', - '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + u'ignore', + u'^Module backports was already imported from (.*), but (.*) is being added to sys.path$', UserWarning ) @@ -37,7 +39,7 @@ def __define_global_system_encoding_variable__(): # and reset to None encoding = None - if not sys.platform.startswith('win') and sys.stdin is not None: + if not sys.platform.startswith(u'win') and sys.stdin is not None: # On linux we can rely on sys.stdin for the encoding since it # most commonly matches the filesystem encoding. This however # does not apply to windows @@ -63,16 +65,16 @@ def __define_global_system_encoding_variable__(): # the way back to ascii encoding = sys.getdefaultencoding() if not encoding: - if sys.platform.startswith('darwin'): + if sys.platform.startswith(u'darwin'): # Mac OS X uses UTF-8 - encoding = 'utf-8' - elif sys.platform.startswith('win'): + encoding = u'utf-8' + elif sys.platform.startswith(u'win'): # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” # to refer to whatever the currently configured encoding is. - encoding = 'mbcs' + encoding = u'mbcs' else: # On linux default to ascii as a last resort - encoding = 'ascii' + encoding = u'ascii' # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: @@ -83,7 +85,7 @@ def __define_global_system_encoding_variable__(): import builtins # pylint: disable=import-error # Define the detected encoding as a built-in variable for ease of use - setattr(builtins, '__salt_system_encoding__', encoding) + setattr(builtins, u'__salt_system_encoding__', encoding) # This is now garbage collectable del sys diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3af3dce..ce6c562 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -30,7 +30,7 @@ from salt.version import __version__ # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) From 16614c462df131a69d76ec5a9a539fa74c5b6790 Mon Sep 17 00:00:00 2001 From: Vlad Paciu Date: Fri, 11 Aug 2017 09:35:09 -0400 Subject: [PATCH 543/769] fix salt consul.get module crush on empty keys --- src/saltext/consul/modules/consul.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 33d746c..9ec09f1 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -205,7 +205,11 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): if ret['res']: if decode: for item in ret['data']: - item['Value'] = base64.b64decode(item['Value']) + if item['Value'] != None: + item['Value'] = base64.b64decode(item['Value']) + else: + item['Value'] = "" + return ret From ea2805d85aabac6d5ed68b8d7069ec20109f1109 Mon Sep 17 00:00:00 2001 From: Vlad Paciu Date: Fri, 11 Aug 2017 15:31:38 -0400 Subject: [PATCH 544/769] fix idents --- src/saltext/consul/modules/consul.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 9ec09f1..68a4bd3 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -206,9 +206,9 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): if decode: for item in ret['data']: if item['Value'] != None: - item['Value'] = base64.b64decode(item['Value']) + item['Value'] = base64.b64decode(item['Value']) else: - item['Value'] = "" + item['Value'] = "" return ret From 1b9376e102b46f4bc9de903fa5cde35b6fe2b675 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 14 Aug 2017 09:01:08 -0700 Subject: [PATCH 545/769] Updating all instances of check_minions. --- src/saltext/consul/pillar/consul_pillar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 0d10b80..d661649 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -167,7 +167,8 @@ def ext_pillar(minion_id, opts['target'] = match.group(1) temp = temp.replace(match.group(0), '') checker = salt.utils.minions.CkMinions(__opts__) - minions = checker.check_minions(opts['target'], 'compound') + _res = checker.check_minions(opts['target'], 'compound') + minions = _res['minions'] if minion_id not in minions: return {} From bb0566afdf871d56f79f959f375b662965e929f8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 17 Aug 2017 14:04:30 -0500 Subject: [PATCH 546/769] Normalize the salt caching API https://github.com/saltstack/salt/pull/40429 changed the attribute used in `salt/cache/__init__.py` without considering the usage of the cache system elsewhere in the codebase. This led to a cascade of additional changes to try to make everything work correctly. This commit does 2 things: 1) Reverts the calls to the list function to use `list` instead of `ls` to conform with the established API used elsewhere in the codebase. 2) Modifies the cache modules so that they now use `list_` functions, aliased to `list` so that calls to cache loader instances' `*.list` functions elsewhere in the codebase will work. --- src/saltext/consul/cache/__init__.py | 6 ++---- src/saltext/consul/cache/consul.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 4c66dd3..94d7a36 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -224,7 +224,7 @@ def flush(self, bank, key=None): fun = '{0}.flush'.format(self.driver) return self.modules[fun](bank, key=key, **self._kwargs) - def ls(self, bank): + def list(self, bank): ''' Lists entries stored in the specified bank. @@ -240,11 +240,9 @@ def ls(self, bank): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.ls'.format(self.driver) + fun = '{0}.list'.format(self.driver) return self.modules[fun](bank, **self._kwargs) - list = ls - def contains(self, bank, key=None): ''' Checks if the specified bank contains the specified key. diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index b545c96..d8f1b32 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -61,7 +61,7 @@ # Define the module's virtual name __virtualname__ = 'consul' -__func_alias__ = {'list': 'ls'} +__func_alias__ = {'list_': 'list'} def __virtual__(): @@ -139,7 +139,7 @@ def flush(bank, key=None): ) -def ls(bank): +def list_(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' From 4fb745185d319645377156df660c0ba3c1059590 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 12 May 2017 13:29:48 -0600 Subject: [PATCH 547/769] Remove deprecated Legacy Git Pillar code --- src/saltext/consul/pillar/__init__.py | 46 ++++++++++----------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index ce6c562..f93f4ee 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -233,7 +233,7 @@ def fetch_pillar(self): functions=self.functions, pillar_override=self.pillar_override, pillarenv=self.pillarenv) - return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here + return fresh_pillar.compile_pillar() def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) @@ -763,7 +763,7 @@ def render_pillar(self, matches, errors=None): return pillar, errors - def _external_pillar_data(self, pillar, val, pillar_dirs, key): + def _external_pillar_data(self, pillar, val, key): ''' Builds actual pillar data structure and updates the ``pillar`` variable ''' @@ -772,26 +772,16 @@ def _external_pillar_data(self, pillar, val, pillar_dirs, key): if isinstance(val, dict): ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - if key == 'git': - ext = self.ext_pillars[key](self.minion_id, - val, - pillar_dirs) - else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - *val) + ext = self.ext_pillars[key](self.minion_id, + pillar, + *val) else: - if key == 'git': - ext = self.ext_pillars[key](self.minion_id, - val, - pillar_dirs) - else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - val) + ext = self.ext_pillars[key](self.minion_id, + pillar, + val) return ext - def ext_pillar(self, pillar, pillar_dirs, errors=None): + def ext_pillar(self, pillar, errors=None): ''' Render the external pillar data ''' @@ -843,9 +833,8 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): continue try: ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) + val, + key) except Exception as exc: errors.append( 'Failed to load ext_pillar {0}: {1}'.format( @@ -867,16 +856,14 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): ext = None return pillar, errors - def compile_pillar(self, ext=True, pillar_dirs=None): + def compile_pillar(self, ext=True): ''' Render the pillar data and return ''' top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'], errors = self.ext_pillar( - self.pillar_override, - pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar(self.pillar_override) self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) @@ -888,8 +875,7 @@ def compile_pillar(self, ext=True, pillar_dirs=None): else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar, errors = self.ext_pillar( - pillar, pillar_dirs, errors=errors) + pillar, errors = self.ext_pillar(pillar, errors=errors) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -984,6 +970,6 @@ def decrypt_pillar(self, pillar): # ext_pillar etc. class AsyncPillar(Pillar): @tornado.gen.coroutine - def compile_pillar(self, ext=True, pillar_dirs=None): - ret = super(AsyncPillar, self).compile_pillar(ext=ext, pillar_dirs=pillar_dirs) + def compile_pillar(self, ext=True): + ret = super(AsyncPillar, self).compile_pillar(ext=ext) raise tornado.gen.Return(ret) From bf6d76ed487dcf0e492ae352e011eb6bdb34a765 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Fri, 28 Jul 2017 13:34:07 -0400 Subject: [PATCH 548/769] Added RemotePillarMixin containing remote pillar common functionality - retrieve the config data to be sent to external pillar functions --- src/saltext/consul/pillar/__init__.py | 50 ++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f93f4ee..168428a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -23,6 +23,7 @@ import salt.utils.url import salt.utils.cache import salt.utils.crypt +import salt.utils.dictupdate from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -72,7 +73,54 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None 'local': AsyncPillar, }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) + + +class RemotePillarMixin(object): + ''' + Common remote pillar functionality + ''' + def get_ext_pillar_extra_minion_data(self, opts): + ''' + Returns the extra data from the minion's opts dict (the config file). + + This data will be passed to external pillar functions. + ''' + def get_subconfig(opts_key): + ''' + Returns a dict containing the opts key subtree, while maintaining + the opts structure + ''' + ret_dict = aux_dict = {} + config_val = opts + subkeys = opts_key.split(':') + # Build an empty dict with the opts path + for subkey in subkeys[:-1]: + aux_dict[subkey] = {} + aux_dict = aux_dict[subkey] + if not config_val.get(subkey): + # The subkey is not in the config + return {} + config_val = config_val[subkey] + if subkeys[-1] not in config_val: + return {} + aux_dict[subkeys[-1]] = config_val[subkeys[-1]] + return ret_dict + + extra_data = {} + if 'pass_to_ext_pillars' in opts: + if not isinstance(opts['pass_to_ext_pillars'], list): + log.exception('\'pass_to_ext_pillars\' config is malformed.') + raise SaltClientError('\'pass_to_ext_pillars\' config is ' + 'malformed.') + for key in opts['pass_to_ext_pillars']: + salt.utils.dictupdate.update(extra_data, + get_subconfig(key), + recursive_update=True, + merge_lists=True) + log.trace('ext_pillar_extra_data = {0}'.format(extra_data)) + return pillar_override class AsyncRemotePillar(object): From d28201207f00d7e731509bc5823a3c85efb5a320 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Mon, 21 Aug 2017 06:35:38 -0400 Subject: [PATCH 549/769] Inherited RemotePillarMixin and added sending the extra minion data when requesting the pillar --- src/saltext/consul/pillar/__init__.py | 28 ++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 168428a..13a4a5d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -120,15 +120,15 @@ def get_subconfig(opts_key): recursive_update=True, merge_lists=True) log.trace('ext_pillar_extra_data = {0}'.format(extra_data)) - return pillar_override + return extra_data -class AsyncRemotePillar(object): +class AsyncRemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -141,6 +141,14 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) @tornado.gen.coroutine def compile_pillar(self): @@ -152,6 +160,7 @@ def compile_pillar(self): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -174,12 +183,12 @@ def compile_pillar(self): raise tornado.gen.Return(ret_pillar) -class RemotePillar(object): +class RemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -192,6 +201,14 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) def compile_pillar(self): ''' @@ -202,6 +219,7 @@ def compile_pillar(self): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: From 7e339e2536052e3cf0b5efc60d6ee47b8910fe30 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Mon, 21 Aug 2017 06:44:18 -0400 Subject: [PATCH 550/769] Passed extra minion data (if with content) to external pillar functions --- src/saltext/consul/pillar/__init__.py | 46 ++++++++++++++++++++------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 13a4a5d..e5e5dbb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -37,7 +37,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -56,12 +56,14 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, + extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -253,7 +255,7 @@ class PillarCache(object): ''' # TODO ABC? def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -331,7 +333,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.minion_id = minion_id self.ext = ext if pillarenv is None: @@ -377,6 +379,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') def __valid_on_demand_ext_pillar(self, opts): ''' @@ -836,15 +842,31 @@ def _external_pillar_data(self, pillar, val, key): ext = None if isinstance(val, dict): - ext = self.ext_pillars[key](self.minion_id, pillar, **val) + if self.extra_minion_data: + ext = self.ext_pillars[key](self.minion_id, pillar, + self.extra_minion_data, **val) + else: + ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - ext = self.ext_pillars[key](self.minion_id, - pillar, - *val) + if self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, *val, + extra_minion_data=self.extra_minion_data) + else: + ext = self.ext_pillars[key](self.minion_id, + pillar, + *val) else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - val) + if self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, + pillar, + val, + extra_minion_data=self.extra_minion_data) + else: + ext = self.ext_pillars[key](self.minion_id, + pillar, + val) return ext def ext_pillar(self, pillar, errors=None): From 698c3611beadc3b681856b816c8f8fa20722f356 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Mon, 21 Aug 2017 08:24:06 -0400 Subject: [PATCH 551/769] Added argspec check to see if external pillar functions support the extra_minion_data parameter --- src/saltext/consul/pillar/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e5e5dbb..76ae1c4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -13,6 +13,7 @@ import tornado.gen import sys import traceback +import inspect # Import salt libs import salt.loader @@ -840,15 +841,17 @@ def _external_pillar_data(self, pillar, val, key): Builds actual pillar data structure and updates the ``pillar`` variable ''' ext = None + args = inspect.getargspec(self.ext_pillars[key]).args if isinstance(val, dict): - if self.extra_minion_data: - ext = self.ext_pillars[key](self.minion_id, pillar, - self.extra_minion_data, **val) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, + extra_minion_data=self.extra_minion_data, **val) else: ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - if self.extra_minion_data: + if ('extra_minion_data' in args) and self.extra_minion_data: ext = self.ext_pillars[key]( self.minion_id, pillar, *val, extra_minion_data=self.extra_minion_data) @@ -857,7 +860,7 @@ def _external_pillar_data(self, pillar, val, key): pillar, *val) else: - if self.extra_minion_data: + if ('extra_minion_data' in args) and self.extra_minion_data: ext = self.ext_pillars[key]( self.minion_id, pillar, From 9f671d10c1abb4d665833662062119ffd25a4052 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Sun, 20 Aug 2017 09:06:39 -0400 Subject: [PATCH 552/769] Prevent spurious "Template does not exist" error This was merged previously (though slightly differently) in #39516 Took me a second to track it down and then realized that I fixed this in 2016.x --- src/saltext/consul/pillar/__init__.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a62e11d..8d5eb7e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -405,20 +405,19 @@ def get_tops(self): self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": From f117646594b7bfe3d50f0c0d47798d3071d8a567 Mon Sep 17 00:00:00 2001 From: John Jawed Date: Fri, 1 Sep 2017 18:33:28 -0700 Subject: [PATCH 553/769] Fix #43295, better handling of consul initialization issues --- src/saltext/consul/cache/consul.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index b545c96..0148c73 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -4,6 +4,8 @@ .. versionadded:: 2016.11.2 +:depends: python-consul >= 0.2.0 + It is up to the system administrator to set up and configure the Consul infrastructure. All is needed for this plugin is a working Consul agent with a read-write access to the key-value store. @@ -81,8 +83,11 @@ def __virtual__(): 'verify': __opts__.get('consul.verify', True), } - global api - api = consul.Consul(**consul_kwargs) + try: + global api + api = consul.Consul(**consul_kwargs) + except AttributeError: + return (False, "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed") return __virtualname__ From a858d7bfd6c14c24ab4e6be05cc4c85dd5dd6204 Mon Sep 17 00:00:00 2001 From: John Jawed Date: Fri, 1 Sep 2017 18:33:28 -0700 Subject: [PATCH 554/769] Fix #43295, better handling of consul initialization issues --- src/saltext/consul/cache/consul.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index b545c96..0148c73 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -4,6 +4,8 @@ .. versionadded:: 2016.11.2 +:depends: python-consul >= 0.2.0 + It is up to the system administrator to set up and configure the Consul infrastructure. All is needed for this plugin is a working Consul agent with a read-write access to the key-value store. @@ -81,8 +83,11 @@ def __virtual__(): 'verify': __opts__.get('consul.verify', True), } - global api - api = consul.Consul(**consul_kwargs) + try: + global api + api = consul.Consul(**consul_kwargs) + except AttributeError: + return (False, "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed") return __virtualname__ From 9b7e761a92afaa65ecaf4a9da3e9fa041a16d80f Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 6 Sep 2017 11:46:03 -0400 Subject: [PATCH 555/769] Changed inspect.getargspec to salt.utils.args.get_function_argspec --- src/saltext/consul/pillar/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 76ae1c4..2840df9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -25,6 +25,7 @@ import salt.utils.cache import salt.utils.crypt import salt.utils.dictupdate +import salt.utils.args from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -841,7 +842,7 @@ def _external_pillar_data(self, pillar, val, key): Builds actual pillar data structure and updates the ``pillar`` variable ''' ext = None - args = inspect.getargspec(self.ext_pillars[key]).args + args = salt.utils.args.get_function_argspec(self.ext_pillars[key]).args if isinstance(val, dict): if ('extra_minion_data' in args) and self.extra_minion_data: From c069e4f82c3cd42690dd37da6a5a0aece9a09fe7 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Sun, 20 Aug 2017 09:06:39 -0400 Subject: [PATCH 556/769] Prevent spurious "Template does not exist" error This was merged previously (though slightly differently) in #39516 Took me a second to track it down and then realized that I fixed this in 2016.x --- src/saltext/consul/pillar/__init__.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a62e11d..8d5eb7e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -405,20 +405,19 @@ def get_tops(self): self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": From 8cec91db55597453ba9e878172940a28b06684d7 Mon Sep 17 00:00:00 2001 From: rtx3 Date: Fri, 8 Sep 2017 10:15:31 +0800 Subject: [PATCH 557/769] 1. Add ACL token to consul module when consul set default ACL rule to deny. 2. Add nodemeta to function catalog_register. 3. Fix a bug when fucntion catalog_register return true buf actually not registered. --- src/saltext/consul/modules/consul.py | 238 +++++++++++++++++++-------- 1 file changed, 170 insertions(+), 68 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 68a4bd3..09fe922 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -42,9 +42,16 @@ def _get_config(): return __salt__['config.get']('consul.url') or \ __salt__['config.get']('consul:url') +def _get_token(): + ''' + Retrieve Consul configuration + ''' + return __salt__['config.get']('consul.token') or \ + __salt__['config.get']('consul:token') def _query(function, consul_url, + token=None, method='GET', api_version='v1', data=None, @@ -59,13 +66,17 @@ def _query(function, :param data: The data to be sent for POST method. :return: The json response from the API call or False. ''' - headers = {} + if not query_params: query_params = {} ret = {'data': '', 'res': True} + if not token: + token = _get_token() + + headers = {"X-Consul-Token": token,"Content-Type": "application/json"} base_url = _urljoin(consul_url, '{0}/'.format(api_version)) url = _urljoin(base_url, function, False) @@ -101,7 +112,7 @@ def _query(function, return ret -def list_(consul_url=None, key=None, **kwargs): +def list_(consul_url=None,token=None, key=None, **kwargs): ''' List keys in Consul @@ -119,6 +130,8 @@ def list_(consul_url=None, key=None, **kwargs): ''' ret = {} + + if not consul_url: consul_url = _get_config() if not consul_url: @@ -140,14 +153,15 @@ def list_(consul_url=None, key=None, **kwargs): function = 'kv/{0}'.format(key) query_params['keys'] = 'True' - + query_params['separator'] = '/' ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): +def get(consul_url=None, key=None, token=None,recurse=False, decode=False, raw=False): ''' Get key from Consul @@ -200,20 +214,17 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): query_params['raw'] = True ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) if ret['res']: if decode: for item in ret['data']: - if item['Value'] != None: - item['Value'] = base64.b64decode(item['Value']) - else: - item['Value'] = "" - + item['Value'] = base64.b64decode(item['Value']) return ret -def put(consul_url=None, key=None, value=None, **kwargs): +def put(consul_url=None, token=None,key=None, value=None, **kwargs): ''' Put values into Consul @@ -321,6 +332,7 @@ def put(consul_url=None, key=None, value=None, **kwargs): function = 'kv/{0}'.format(key) method = 'PUT' ret = _query(consul_url=consul_url, + token=token, function=function, method=method, data=data, @@ -335,7 +347,7 @@ def put(consul_url=None, key=None, value=None, **kwargs): return ret -def delete(consul_url=None, key=None, **kwargs): +def delete(consul_url=None, token=None, key=None, **kwargs): ''' Delete values from Consul @@ -383,6 +395,7 @@ def delete(consul_url=None, key=None, **kwargs): function = 'kv/{0}'.format(key) ret = _query(consul_url=consul_url, + token=token, function=function, method='DELETE', query_params=query_params) @@ -396,7 +409,7 @@ def delete(consul_url=None, key=None, **kwargs): return ret -def agent_checks(consul_url=None): +def agent_checks(consul_url=None,token=None): ''' Returns the checks the local agent is managing @@ -422,11 +435,12 @@ def agent_checks(consul_url=None): function = 'agent/checks' ret = _query(consul_url=consul_url, function=function, + token=token, method='GET') return ret -def agent_services(consul_url=None): +def agent_services(consul_url=None,token=None): ''' Returns the services the local agent is managing @@ -452,11 +466,12 @@ def agent_services(consul_url=None): function = 'agent/services' ret = _query(consul_url=consul_url, function=function, + token=token, method='GET') return ret -def agent_members(consul_url=None, **kwargs): +def agent_members(consul_url=None, token=None, **kwargs): ''' Returns the members as seen by the local serf agent @@ -486,12 +501,13 @@ def agent_members(consul_url=None, **kwargs): function = 'agent/members' ret = _query(consul_url=consul_url, function=function, + token=token, method='GET', query_params=query_params) return ret -def agent_self(consul_url=None): +def agent_self(consul_url=None,token=None): ''' Returns the local node configuration @@ -518,12 +534,13 @@ def agent_self(consul_url=None): function = 'agent/self' ret = _query(consul_url=consul_url, function=function, + token=token, method='GET', query_params=query_params) return ret -def agent_maintenance(consul_url=None, **kwargs): +def agent_maintenance(consul_url=None,token=None, **kwargs): ''' Manages node maintenance mode @@ -567,6 +584,7 @@ def agent_maintenance(consul_url=None, **kwargs): function = 'agent/maintenance' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', query_params=query_params) if res['res']: @@ -579,7 +597,7 @@ def agent_maintenance(consul_url=None, **kwargs): return ret -def agent_join(consul_url=None, address=None, **kwargs): +def agent_join(consul_url=None,token=None, address=None, **kwargs): ''' Triggers the local agent to join a node @@ -614,6 +632,7 @@ def agent_join(consul_url=None, address=None, **kwargs): function = 'agent/join/{0}'.format(address) res = _query(consul_url=consul_url, function=function, + token=token, method='GET', query_params=query_params) if res['res']: @@ -625,7 +644,7 @@ def agent_join(consul_url=None, address=None, **kwargs): return ret -def agent_leave(consul_url=None, node=None): +def agent_leave(consul_url=None,token=None, node=None): ''' Used to instruct the agent to force a node into the left state. @@ -656,6 +675,7 @@ def agent_leave(consul_url=None, node=None): function = 'agent/force-leave/{0}'.format(node) res = _query(consul_url=consul_url, function=function, + token=token, method='GET', query_params=query_params) if res['res']: @@ -667,7 +687,7 @@ def agent_leave(consul_url=None, node=None): return ret -def agent_check_register(consul_url=None, **kwargs): +def agent_check_register(consul_url=None,token=None, **kwargs): ''' The register endpoint is used to add a new check to the local agent. @@ -710,7 +730,7 @@ def agent_check_register(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') - if True not in [True for item in ('script', 'http') if item in kwargs]: + if True not in [True for item in ('script', 'http','ttl') if item in kwargs]: ret['message'] = 'Required parameter "script" or "http" is missing.' ret['res'] = False return ret @@ -737,11 +757,16 @@ def agent_check_register(consul_url=None, **kwargs): data['HTTP'] = kwargs['http'] data['Interval'] = kwargs['interval'] + if 'ttl' in kwargs: + data['TTL'] = kwargs['ttl'] + function = 'agent/check/register' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) + if res['res']: ret['res'] = True ret['message'] = ('Check {0} added to agent.'.format(kwargs['name'])) @@ -751,7 +776,7 @@ def agent_check_register(consul_url=None, **kwargs): return ret -def agent_check_deregister(consul_url=None, checkid=None): +def agent_check_deregister(consul_url=None,token=None, checkid=None): ''' The agent will take care of deregistering the check from the Catalog. @@ -781,6 +806,7 @@ def agent_check_deregister(consul_url=None, checkid=None): function = 'agent/check/deregister/{0}'.format(checkid) res = _query(consul_url=consul_url, function=function, + token=token, method='GET') if res['res']: ret['res'] = True @@ -791,7 +817,7 @@ def agent_check_deregister(consul_url=None, checkid=None): return ret -def agent_check_pass(consul_url=None, checkid=None, **kwargs): +def agent_check_pass(consul_url=None, token=None,checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to passing and the TTL @@ -829,6 +855,7 @@ def agent_check_pass(consul_url=None, checkid=None, **kwargs): function = 'agent/check/pass/{0}'.format(checkid) res = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params, method='GET') if res['res']: @@ -840,7 +867,7 @@ def agent_check_pass(consul_url=None, checkid=None, **kwargs): return ret -def agent_check_warn(consul_url=None, checkid=None, **kwargs): +def agent_check_warn(consul_url=None, token=None,checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to warning and the TTL @@ -878,6 +905,7 @@ def agent_check_warn(consul_url=None, checkid=None, **kwargs): function = 'agent/check/warn/{0}'.format(checkid) res = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params, method='GET') if res['res']: @@ -889,7 +917,7 @@ def agent_check_warn(consul_url=None, checkid=None, **kwargs): return ret -def agent_check_fail(consul_url=None, checkid=None, **kwargs): +def agent_check_fail(consul_url=None, token=None,checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to critical and the @@ -927,6 +955,7 @@ def agent_check_fail(consul_url=None, checkid=None, **kwargs): function = 'agent/check/fail/{0}'.format(checkid) res = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params, method='GET') if res['res']: @@ -938,7 +967,7 @@ def agent_check_fail(consul_url=None, checkid=None, **kwargs): return ret -def agent_service_register(consul_url=None, **kwargs): +def agent_service_register(consul_url=None,token=None, **kwargs): ''' The used to add a new service, with an optional health check, to the local agent. @@ -1042,6 +1071,7 @@ def agent_service_register(consul_url=None, **kwargs): function = 'agent/service/register' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) if res['res']: @@ -1053,7 +1083,7 @@ def agent_service_register(consul_url=None, **kwargs): return ret -def agent_service_deregister(consul_url=None, serviceid=None): +def agent_service_deregister(consul_url=None, token=None,serviceid=None): ''' Used to remove a service. @@ -1084,6 +1114,7 @@ def agent_service_deregister(consul_url=None, serviceid=None): function = 'agent/service/deregister/{0}'.format(serviceid) res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) if res['res']: @@ -1095,7 +1126,7 @@ def agent_service_deregister(consul_url=None, serviceid=None): return ret -def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): +def agent_service_maintenance(consul_url=None, token=None,serviceid=None, **kwargs): ''' Used to place a service into maintenance mode. @@ -1139,6 +1170,7 @@ def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): function = 'agent/service/maintenance/{0}'.format(serviceid) res = _query(consul_url=consul_url, + token=token, function=function, query_params=query_params) @@ -1153,7 +1185,7 @@ def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): return ret -def session_create(consul_url=None, **kwargs): +def session_create(consul_url=None, token=None,**kwargs): ''' Used to create a session. @@ -1232,6 +1264,7 @@ def session_create(consul_url=None, **kwargs): function = 'session/create' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) @@ -1244,7 +1277,7 @@ def session_create(consul_url=None, **kwargs): return ret -def session_list(consul_url=None, return_list=False, **kwargs): +def session_list(consul_url=None, token=None,return_list=False, **kwargs): ''' Used to list sessions. @@ -1280,6 +1313,7 @@ def session_list(consul_url=None, return_list=False, **kwargs): function = 'session/list' ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) if return_list: @@ -1290,7 +1324,7 @@ def session_list(consul_url=None, return_list=False, **kwargs): return ret -def session_destroy(consul_url=None, session=None, **kwargs): +def session_destroy(consul_url=None, token=None,session=None, **kwargs): ''' Destroy session @@ -1327,6 +1361,7 @@ def session_destroy(consul_url=None, session=None, **kwargs): function = 'session/destroy/{0}'.format(session) res = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) if res['res']: ret['res'] = True @@ -1337,7 +1372,7 @@ def session_destroy(consul_url=None, session=None, **kwargs): return ret -def session_info(consul_url=None, session=None, **kwargs): +def session_info(consul_url=None,token=None, session=None, **kwargs): ''' Information about a session @@ -1374,11 +1409,12 @@ def session_info(consul_url=None, session=None, **kwargs): function = 'session/info/{0}'.format(session) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def catalog_register(consul_url=None, **kwargs): +def catalog_register(consul_url=None,token=None,**kwargs): ''' Registers a new node, service, or check @@ -1412,6 +1448,7 @@ def catalog_register(consul_url=None, **kwargs): ''' ret = {} data = {} + data['NodeMeta'] = {} if not consul_url: consul_url = _get_config() if not consul_url: @@ -1431,12 +1468,22 @@ def catalog_register(consul_url=None, **kwargs): return ret if 'address' in kwargs: - data['Address'] = kwargs['address'] + if isinstance(kwargs['address'], list): + _address = kwargs['address'][0] + else: + _address = kwargs['address'] + data['Address'] = _address else: ret['message'] = 'Required argument address argument is missing.' ret['res'] = False return ret + if 'ip_interfaces' in kwargs: + data['TaggedAddresses'] = {} + for k in kwargs['ip_interfaces']: + if kwargs['ip_interfaces'].get(k): + data['TaggedAddresses'][k] = kwargs['ip_interfaces'][k][0] + if 'service' in kwargs: data['Service'] = {} data['Service']['Service'] = kwargs['service'] @@ -1456,6 +1503,42 @@ def catalog_register(consul_url=None, **kwargs): _tags = [_tags] data['Service']['Tags'] = _tags + if 'cpu' in kwargs: + data['NodeMeta']['Cpu'] = kwargs['cpu'] + + if 'num_cpus' in kwargs: + data['NodeMeta']['Cpu_num'] = kwargs['num_cpus'] + + if 'mem' in kwargs: + data['NodeMeta']['Memory'] = kwargs['mem'] + + if 'oscode' in kwargs: + data['NodeMeta']['Os'] = kwargs['oscode'] + + if 'osarch' in kwargs: + data['NodeMeta']['Osarch'] = kwargs['osarch'] + + if 'kernel' in kwargs: + data['NodeMeta']['Kernel'] = kwargs['kernel'] + + if 'kernelrelease' in kwargs: + data['NodeMeta']['Kernelrelease'] = kwargs['kernelrelease'] + + if 'localhost' in kwargs: + data['NodeMeta']['localhost'] = kwargs['localhost'] + + if 'nodename' in kwargs: + data['NodeMeta']['nodename'] = kwargs['nodename'] + + if 'os_family' in kwargs: + data['NodeMeta']['os_family'] = kwargs['os_family'] + + if 'lsb_distrib_description' in kwargs: + data['NodeMeta']['lsb_distrib_description'] = kwargs['lsb_distrib_description'] + + if 'master' in kwargs: + data['NodeMeta']['master'] = kwargs['master'] + if 'check' in kwargs: data['Check'] = {} data['Check']['Name'] = kwargs['check'] @@ -1479,20 +1562,23 @@ def catalog_register(consul_url=None, **kwargs): function = 'catalog/register' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) - if res['res']: + if res['res'] and res['data'] is True: ret['res'] = True ret['message'] = ('Catalog registration ' 'for {0} successful.'.format(kwargs['node'])) else: ret['res'] = False ret['message'] = ('Catalog registration ' - 'for {0} failed.'.format(kwargs['node'])) + 'for {0} failed {1}.'.format(kwargs['node'],res['data']['error'])) + ret['data'] = data return ret -def catalog_deregister(consul_url=None, **kwargs): + +def catalog_deregister(consul_url=None,token=None, **kwargs): ''' Deregisters a node, service, or check @@ -1541,19 +1627,22 @@ def catalog_deregister(consul_url=None, **kwargs): function = 'catalog/deregister' res = _query(consul_url=consul_url, function=function, + token=token, method='PUT', data=data) - if res['res']: + + if res['res'] and res['data'] is True: ret['res'] = True - ret['message'] = 'Catalog item {0} removed.'.format(kwargs['node']) + ret['message'] = ('Catalog deregistration ' + 'for {0} successful.'.format(kwargs['node'])) else: ret['res'] = False - ret['message'] = ('Removing Catalog ' - 'item {0} failed.'.format(kwargs['node'])) + ret['message'] = ('Catalog deregistration ' + 'for {0} failed {1}.'.format(kwargs['node'],res['data']['error'])) return ret -def catalog_datacenters(consul_url=None): +def catalog_datacenters(consul_url=None,token=None): ''' Return list of available datacenters from catalog. @@ -1578,11 +1667,12 @@ def catalog_datacenters(consul_url=None): function = 'catalog/datacenters' ret = _query(consul_url=consul_url, - function=function) + function=function, + token=token) return ret -def catalog_nodes(consul_url=None, **kwargs): +def catalog_nodes(consul_url=None,token=None, **kwargs): ''' Return list of available nodes from catalog. @@ -1614,11 +1704,12 @@ def catalog_nodes(consul_url=None, **kwargs): function = 'catalog/nodes' ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def catalog_services(consul_url=None, **kwargs): +def catalog_services(consul_url=None,token=None, **kwargs): ''' Return list of available services rom catalog. @@ -1650,11 +1741,12 @@ def catalog_services(consul_url=None, **kwargs): function = 'catalog/services' ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def catalog_service(consul_url=None, service=None, **kwargs): +def catalog_service(consul_url=None, token=None,service=None, **kwargs): ''' Information about the registered service. @@ -1693,11 +1785,12 @@ def catalog_service(consul_url=None, service=None, **kwargs): function = 'catalog/service/{0}'.format(service) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def catalog_node(consul_url=None, node=None, **kwargs): +def catalog_node(consul_url=None,token=None, node=None, **kwargs): ''' Information about the registered node. @@ -1733,11 +1826,12 @@ def catalog_node(consul_url=None, node=None, **kwargs): function = 'catalog/node/{0}'.format(node) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def health_node(consul_url=None, node=None, **kwargs): +def health_node(consul_url=None, token=None,node=None, **kwargs): ''' Health information about the registered node. @@ -1773,11 +1867,12 @@ def health_node(consul_url=None, node=None, **kwargs): function = 'health/node/{0}'.format(node) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def health_checks(consul_url=None, service=None, **kwargs): +def health_checks(consul_url=None,token=None, service=None, **kwargs): ''' Health information about the registered service. @@ -1813,11 +1908,12 @@ def health_checks(consul_url=None, service=None, **kwargs): function = 'health/checks/{0}'.format(service) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def health_service(consul_url=None, service=None, **kwargs): +def health_service(consul_url=None, token=None,service=None, **kwargs): ''' Health information about the registered service. @@ -1864,11 +1960,12 @@ def health_service(consul_url=None, service=None, **kwargs): function = 'health/service/{0}'.format(service) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def health_state(consul_url=None, state=None, **kwargs): +def health_state(consul_url=None,token=None, state=None, **kwargs): ''' Returns the checks in the state provided on the path. @@ -1914,11 +2011,12 @@ def health_state(consul_url=None, state=None, **kwargs): function = 'health/state/{0}'.format(state) ret = _query(consul_url=consul_url, function=function, + token=token, query_params=query_params) return ret -def status_leader(consul_url=None): +def status_leader(consul_url=None,token=None): ''' Returns the current Raft leader @@ -1943,11 +2041,12 @@ def status_leader(consul_url=None): function = 'status/leader' ret = _query(consul_url=consul_url, - function=function) + function=function, + token=token) return ret -def status_peers(consul_url): +def status_peers(consul_url,token=None): ''' Returns the current Raft peer set @@ -1973,17 +2072,16 @@ def status_peers(consul_url): function = 'status/peers' ret = _query(consul_url=consul_url, - function=function) + function=function, + token=token) return ret -def acl_create(consul_url=None, **kwargs): +def acl_create(consul_url=None,token=None, **kwargs): ''' Create a new ACL token. :param consul_url: The Consul server URL. - :param id: Unique identifier for the ACL to create - leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2014,9 +2112,6 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') - if 'id' in kwargs: - data['ID'] = kwargs['id'] - if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2025,6 +2120,7 @@ def acl_create(consul_url=None, **kwargs): function = 'acl/create' res = _query(consul_url=consul_url, + token=token, data=data, method='PUT', function=function) @@ -2039,7 +2135,7 @@ def acl_create(consul_url=None, **kwargs): return ret -def acl_update(consul_url=None, **kwargs): +def acl_update(consul_url=None, token=None, **kwargs): ''' Update an ACL token. @@ -2090,6 +2186,7 @@ def acl_update(consul_url=None, **kwargs): function = 'acl/update' res = _query(consul_url=consul_url, + token=token, data=data, method='PUT', function=function) @@ -2105,7 +2202,7 @@ def acl_update(consul_url=None, **kwargs): return ret -def acl_delete(consul_url=None, **kwargs): +def acl_delete(consul_url=None,token=None, **kwargs): ''' Delete an ACL token. @@ -2135,8 +2232,9 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/destroy/{0}'.format(kwargs['id']) + function = 'acl/delete/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, + token=token, data=data, method='PUT', function=function) @@ -2190,7 +2288,7 @@ def acl_info(consul_url=None, **kwargs): return ret -def acl_clone(consul_url=None, **kwargs): +def acl_clone(consul_url=None,token=None, **kwargs): ''' Information about an ACL token. @@ -2223,6 +2321,7 @@ def acl_clone(consul_url=None, **kwargs): function = 'acl/clone/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, + token=token, data=data, method='PUT', function=function) @@ -2237,7 +2336,7 @@ def acl_clone(consul_url=None, **kwargs): return ret -def acl_list(consul_url=None, **kwargs): +def acl_list(consul_url=None, token=None,**kwargs): ''' List the ACL tokens. @@ -2268,13 +2367,14 @@ def acl_list(consul_url=None, **kwargs): function = 'acl/list' ret = _query(consul_url=consul_url, + token=token, data=data, method='PUT', function=function) return ret -def event_fire(consul_url=None, name=None, **kwargs): +def event_fire(consul_url=None,token=None, name=None, **kwargs): ''' List the ACL tokens. @@ -2321,6 +2421,7 @@ def event_fire(consul_url=None, name=None, **kwargs): function = 'event/fire/{0}'.format(name) res = _query(consul_url=consul_url, + token=token, query_params=query_params, method='PUT', function=function) @@ -2336,7 +2437,7 @@ def event_fire(consul_url=None, name=None, **kwargs): return ret -def event_list(consul_url=None, **kwargs): +def event_list(consul_url=None, token=None,**kwargs): ''' List the recent events. @@ -2368,6 +2469,7 @@ def event_list(consul_url=None, **kwargs): function = 'event/list/' ret = _query(consul_url=consul_url, + token=token, query_params=query_params, function=function) - return ret + return ret \ No newline at end of file From aba04678c5bc1fcb72c1e2109498afb05b7f9cd7 Mon Sep 17 00:00:00 2001 From: rtx3 Date: Thu, 21 Sep 2017 17:19:05 +0800 Subject: [PATCH 558/769] fix lint issue --- src/saltext/consul/modules/consul.py | 105 +++++++++++++-------------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 09fe922..75c7542 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -42,6 +42,7 @@ def _get_config(): return __salt__['config.get']('consul.url') or \ __salt__['config.get']('consul:url') + def _get_token(): ''' Retrieve Consul configuration @@ -49,6 +50,7 @@ def _get_token(): return __salt__['config.get']('consul.token') or \ __salt__['config.get']('consul:token') + def _query(function, consul_url, token=None, @@ -76,7 +78,7 @@ def _query(function, if not token: token = _get_token() - headers = {"X-Consul-Token": token,"Content-Type": "application/json"} + headers = {"X-Consul-Token": token, "Content-Type": "application/json"} base_url = _urljoin(consul_url, '{0}/'.format(api_version)) url = _urljoin(base_url, function, False) @@ -112,7 +114,7 @@ def _query(function, return ret -def list_(consul_url=None,token=None, key=None, **kwargs): +def list_(consul_url=None, token=None, key=None, **kwargs): ''' List keys in Consul @@ -131,7 +133,6 @@ def list_(consul_url=None,token=None, key=None, **kwargs): ''' ret = {} - if not consul_url: consul_url = _get_config() if not consul_url: @@ -161,7 +162,7 @@ def list_(consul_url=None,token=None, key=None, **kwargs): return ret -def get(consul_url=None, key=None, token=None,recurse=False, decode=False, raw=False): +def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw=False): ''' Get key from Consul @@ -224,7 +225,7 @@ def get(consul_url=None, key=None, token=None,recurse=False, decode=False, raw=F return ret -def put(consul_url=None, token=None,key=None, value=None, **kwargs): +def put(consul_url=None, token=None, key=None, value=None, **kwargs): ''' Put values into Consul @@ -409,7 +410,7 @@ def delete(consul_url=None, token=None, key=None, **kwargs): return ret -def agent_checks(consul_url=None,token=None): +def agent_checks(consul_url=None, token=None): ''' Returns the checks the local agent is managing @@ -440,7 +441,7 @@ def agent_checks(consul_url=None,token=None): return ret -def agent_services(consul_url=None,token=None): +def agent_services(consul_url=None, token=None): ''' Returns the services the local agent is managing @@ -507,7 +508,7 @@ def agent_members(consul_url=None, token=None, **kwargs): return ret -def agent_self(consul_url=None,token=None): +def agent_self(consul_url=None, token=None): ''' Returns the local node configuration @@ -540,7 +541,7 @@ def agent_self(consul_url=None,token=None): return ret -def agent_maintenance(consul_url=None,token=None, **kwargs): +def agent_maintenance(consul_url=None, token=None, **kwargs): ''' Manages node maintenance mode @@ -597,7 +598,7 @@ def agent_maintenance(consul_url=None,token=None, **kwargs): return ret -def agent_join(consul_url=None,token=None, address=None, **kwargs): +def agent_join(consul_url=None, token=None, address=None, **kwargs): ''' Triggers the local agent to join a node @@ -644,7 +645,7 @@ def agent_join(consul_url=None,token=None, address=None, **kwargs): return ret -def agent_leave(consul_url=None,token=None, node=None): +def agent_leave(consul_url=None, token=None, node=None): ''' Used to instruct the agent to force a node into the left state. @@ -687,7 +688,7 @@ def agent_leave(consul_url=None,token=None, node=None): return ret -def agent_check_register(consul_url=None,token=None, **kwargs): +def agent_check_register(consul_url=None, token=None, **kwargs): ''' The register endpoint is used to add a new check to the local agent. @@ -730,7 +731,7 @@ def agent_check_register(consul_url=None,token=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') - if True not in [True for item in ('script', 'http','ttl') if item in kwargs]: + if True not in [True for item in ('script', 'http', 'ttl') if item in kwargs]: ret['message'] = 'Required parameter "script" or "http" is missing.' ret['res'] = False return ret @@ -776,7 +777,7 @@ def agent_check_register(consul_url=None,token=None, **kwargs): return ret -def agent_check_deregister(consul_url=None,token=None, checkid=None): +def agent_check_deregister(consul_url=None, token=None, checkid=None): ''' The agent will take care of deregistering the check from the Catalog. @@ -817,7 +818,7 @@ def agent_check_deregister(consul_url=None,token=None, checkid=None): return ret -def agent_check_pass(consul_url=None, token=None,checkid=None, **kwargs): +def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to passing and the TTL @@ -867,7 +868,7 @@ def agent_check_pass(consul_url=None, token=None,checkid=None, **kwargs): return ret -def agent_check_warn(consul_url=None, token=None,checkid=None, **kwargs): +def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to warning and the TTL @@ -917,7 +918,7 @@ def agent_check_warn(consul_url=None, token=None,checkid=None, **kwargs): return ret -def agent_check_fail(consul_url=None, token=None,checkid=None, **kwargs): +def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): ''' This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to critical and the @@ -967,7 +968,7 @@ def agent_check_fail(consul_url=None, token=None,checkid=None, **kwargs): return ret -def agent_service_register(consul_url=None,token=None, **kwargs): +def agent_service_register(consul_url=None, token=None, **kwargs): ''' The used to add a new service, with an optional health check, to the local agent. @@ -1083,7 +1084,7 @@ def agent_service_register(consul_url=None,token=None, **kwargs): return ret -def agent_service_deregister(consul_url=None, token=None,serviceid=None): +def agent_service_deregister(consul_url=None, token=None, serviceid=None): ''' Used to remove a service. @@ -1126,7 +1127,7 @@ def agent_service_deregister(consul_url=None, token=None,serviceid=None): return ret -def agent_service_maintenance(consul_url=None, token=None,serviceid=None, **kwargs): +def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwargs): ''' Used to place a service into maintenance mode. @@ -1185,7 +1186,7 @@ def agent_service_maintenance(consul_url=None, token=None,serviceid=None, **kwar return ret -def session_create(consul_url=None, token=None,**kwargs): +def session_create(consul_url=None, token=None, **kwargs): ''' Used to create a session. @@ -1277,7 +1278,7 @@ def session_create(consul_url=None, token=None,**kwargs): return ret -def session_list(consul_url=None, token=None,return_list=False, **kwargs): +def session_list(consul_url=None, token=None, return_list=False, **kwargs): ''' Used to list sessions. @@ -1324,7 +1325,7 @@ def session_list(consul_url=None, token=None,return_list=False, **kwargs): return ret -def session_destroy(consul_url=None, token=None,session=None, **kwargs): +def session_destroy(consul_url=None, token=None, session=None, **kwargs): ''' Destroy session @@ -1372,7 +1373,7 @@ def session_destroy(consul_url=None, token=None,session=None, **kwargs): return ret -def session_info(consul_url=None,token=None, session=None, **kwargs): +def session_info(consul_url=None, token=None, session=None, **kwargs): ''' Information about a session @@ -1414,7 +1415,7 @@ def session_info(consul_url=None,token=None, session=None, **kwargs): return ret -def catalog_register(consul_url=None,token=None,**kwargs): +def catalog_register(consul_url=None, token=None, **kwargs): ''' Registers a new node, service, or check @@ -1565,20 +1566,19 @@ def catalog_register(consul_url=None,token=None,**kwargs): token=token, method='PUT', data=data) - if res['res'] and res['data'] is True: + if res['res']: ret['res'] = True ret['message'] = ('Catalog registration ' 'for {0} successful.'.format(kwargs['node'])) else: ret['res'] = False ret['message'] = ('Catalog registration ' - 'for {0} failed {1}.'.format(kwargs['node'],res['data']['error'])) + 'for {0} failed.'.format(kwargs['node'])) ret['data'] = data return ret - -def catalog_deregister(consul_url=None,token=None, **kwargs): +def catalog_deregister(consul_url=None, token=None, **kwargs): ''' Deregisters a node, service, or check @@ -1631,18 +1631,17 @@ def catalog_deregister(consul_url=None,token=None, **kwargs): method='PUT', data=data) - if res['res'] and res['data'] is True: + if res['res']: ret['res'] = True - ret['message'] = ('Catalog deregistration ' - 'for {0} successful.'.format(kwargs['node'])) + ret['message'] = 'Catalog item {0} removed.'.format(kwargs['node']) else: ret['res'] = False - ret['message'] = ('Catalog deregistration ' - 'for {0} failed {1}.'.format(kwargs['node'],res['data']['error'])) + ret['message'] = ('Removing Catalog ' + 'item {0} failed.'.format(kwargs['node'])) return ret -def catalog_datacenters(consul_url=None,token=None): +def catalog_datacenters(consul_url=None, token=None): ''' Return list of available datacenters from catalog. @@ -1672,7 +1671,7 @@ def catalog_datacenters(consul_url=None,token=None): return ret -def catalog_nodes(consul_url=None,token=None, **kwargs): +def catalog_nodes(consul_url=None, token=None, **kwargs): ''' Return list of available nodes from catalog. @@ -1709,7 +1708,7 @@ def catalog_nodes(consul_url=None,token=None, **kwargs): return ret -def catalog_services(consul_url=None,token=None, **kwargs): +def catalog_services(consul_url=None, token=None, **kwargs): ''' Return list of available services rom catalog. @@ -1746,7 +1745,7 @@ def catalog_services(consul_url=None,token=None, **kwargs): return ret -def catalog_service(consul_url=None, token=None,service=None, **kwargs): +def catalog_service(consul_url=None, token=None, service=None, **kwargs): ''' Information about the registered service. @@ -1790,7 +1789,7 @@ def catalog_service(consul_url=None, token=None,service=None, **kwargs): return ret -def catalog_node(consul_url=None,token=None, node=None, **kwargs): +def catalog_node(consul_url=None, token=None, node=None, **kwargs): ''' Information about the registered node. @@ -1831,7 +1830,7 @@ def catalog_node(consul_url=None,token=None, node=None, **kwargs): return ret -def health_node(consul_url=None, token=None,node=None, **kwargs): +def health_node(consul_url=None, token=None, node=None, **kwargs): ''' Health information about the registered node. @@ -1872,7 +1871,7 @@ def health_node(consul_url=None, token=None,node=None, **kwargs): return ret -def health_checks(consul_url=None,token=None, service=None, **kwargs): +def health_checks(consul_url=None, token=None, service=None, **kwargs): ''' Health information about the registered service. @@ -1913,7 +1912,7 @@ def health_checks(consul_url=None,token=None, service=None, **kwargs): return ret -def health_service(consul_url=None, token=None,service=None, **kwargs): +def health_service(consul_url=None, token=None, service=None, **kwargs): ''' Health information about the registered service. @@ -1965,7 +1964,7 @@ def health_service(consul_url=None, token=None,service=None, **kwargs): return ret -def health_state(consul_url=None,token=None, state=None, **kwargs): +def health_state(consul_url=None, token=None, state=None, **kwargs): ''' Returns the checks in the state provided on the path. @@ -2016,7 +2015,7 @@ def health_state(consul_url=None,token=None, state=None, **kwargs): return ret -def status_leader(consul_url=None,token=None): +def status_leader(consul_url=None, token=None): ''' Returns the current Raft leader @@ -2046,7 +2045,7 @@ def status_leader(consul_url=None,token=None): return ret -def status_peers(consul_url,token=None): +def status_peers(consul_url, token=None): ''' Returns the current Raft peer set @@ -2077,7 +2076,7 @@ def status_peers(consul_url,token=None): return ret -def acl_create(consul_url=None,token=None, **kwargs): +def acl_create(consul_url=None, token=None, **kwargs): ''' Create a new ACL token. @@ -2202,7 +2201,7 @@ def acl_update(consul_url=None, token=None, **kwargs): return ret -def acl_delete(consul_url=None,token=None, **kwargs): +def acl_delete(consul_url=None, token=None, **kwargs): ''' Delete an ACL token. @@ -2288,7 +2287,7 @@ def acl_info(consul_url=None, **kwargs): return ret -def acl_clone(consul_url=None,token=None, **kwargs): +def acl_clone(consul_url=None, token=None, **kwargs): ''' Information about an ACL token. @@ -2336,7 +2335,7 @@ def acl_clone(consul_url=None,token=None, **kwargs): return ret -def acl_list(consul_url=None, token=None,**kwargs): +def acl_list(consul_url=None, token=None, **kwargs): ''' List the ACL tokens. @@ -2367,14 +2366,14 @@ def acl_list(consul_url=None, token=None,**kwargs): function = 'acl/list' ret = _query(consul_url=consul_url, - token=token, + token=token, data=data, method='PUT', function=function) return ret -def event_fire(consul_url=None,token=None, name=None, **kwargs): +def event_fire(consul_url=None, token=None, name=None, **kwargs): ''' List the ACL tokens. @@ -2437,7 +2436,7 @@ def event_fire(consul_url=None,token=None, name=None, **kwargs): return ret -def event_list(consul_url=None, token=None,**kwargs): +def event_list(consul_url=None, token=None, **kwargs): ''' List the recent events. @@ -2472,4 +2471,4 @@ def event_list(consul_url=None, token=None,**kwargs): token=token, query_params=query_params, function=function) - return ret \ No newline at end of file + return ret From 4e420b3306b9f4d394b76787520c1dfd23c22acf Mon Sep 17 00:00:00 2001 From: rtx3 Date: Thu, 21 Sep 2017 17:27:33 +0800 Subject: [PATCH 559/769] fix missing item value --- src/saltext/consul/modules/consul.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 75c7542..fba37ce 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -221,7 +221,10 @@ def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw= if ret['res']: if decode: for item in ret['data']: - item['Value'] = base64.b64decode(item['Value']) + if item['Value'] != None: + item['Value'] = base64.b64decode(item['Value']) + else: + item['Value'] = "" return ret From f7be39da95b264374cb6e1d9b08544b94bec6c83 Mon Sep 17 00:00:00 2001 From: vernoncole Date: Thu, 21 Sep 2017 10:26:00 -0600 Subject: [PATCH 560/769] correct default value for salt.cache.Cache --- src/saltext/consul/cache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 94d7a36..fc5e5f0 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -73,7 +73,7 @@ def __init__(self, opts, cachedir=None, **kwargs): self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) else: self.cachedir = cachedir - self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS) + self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS['cache']) self.serial = Serial(opts) self._modules = None self._kwargs = kwargs From ff2cfcfae1dd8a3b6948dabd9cf5250969d6f948 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 21 Sep 2017 23:11:10 +0200 Subject: [PATCH 561/769] make cached pillars use pillarenv rather than saltenv --- src/saltext/consul/pillar/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8d5eb7e..0fbfbfe 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -235,25 +235,25 @@ def fetch_pillar(self): return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs - log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + log.debug('Scanning pillar cache for information about minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) log.debug('Scanning cache: {0}'.format(self.cache._dict)) # Check the cache! if self.minion_id in self.cache: # Keyed by minion_id # TODO Compare grains, etc? - if self.saltenv in self.cache[self.minion_id]: + if self.pillarenv in self.cache[self.minion_id]: # We have a cache hit! Send it back. - log.debug('Pillar cache hit for minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) - return self.cache[self.minion_id][self.saltenv] + log.debug('Pillar cache hit for minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) + return self.cache[self.minion_id][self.pillarenv] else: # We found the minion but not the env. Store it. fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id][self.saltenv] = fresh_pillar - log.debug('Pillar cache miss for saltenv {0} for minion {1}'.format(self.saltenv, self.minion_id)) + self.cache[self.minion_id][self.pillarenv] = fresh_pillar + log.debug('Pillar cache miss for pillarenv {0} for minion {1}'.format(self.pillarenv, self.minion_id)) return fresh_pillar else: # We haven't seen this minion yet in the cache. Store it. fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id] = {self.saltenv: fresh_pillar} + self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} log.debug('Pillar cache miss for minion {0}'.format(self.minion_id)) log.debug('Current pillar cache: {0}'.format(self.cache._dict)) # FIXME hack! return fresh_pillar From c6fb46eccde92f8ad02ade3c4ae35c66acfab556 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 12 Sep 2017 09:23:06 -0500 Subject: [PATCH 562/769] Fix missing PER_REMOTE_ONLY in cache.clear_git_lock runner This allows for the runner to clear the git lock if any of the per-remote-only params was present in the configuration. This also adds PER_REMOTE_ONLY attributes to the winrepo runner and git_pillar external pillar, and fixes all refs to GitBase's init_remotes function to include PER_REMOTE_ONLY arguments. This will insulate us against future headaches if either git_pillar or winrepo gain extra per-remote-only params, as we will only need to make the change to the attribute in their respective modules. --- src/saltext/consul/pillar/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a2d1dcb..2c3bed7 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -763,9 +763,12 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): and self.opts.get('__role') != 'minion': # Avoid circular import import salt.utils.gitfs - from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES + import salt.pillar.git_pillar git_pillar = salt.utils.gitfs.GitPillar(self.opts) - git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES) + git_pillar.init_remotes( + self.ext['git'], + salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + salt.pillar.git_pillar.PER_REMOTE_ONLY) git_pillar.fetch_remotes() except TypeError: # Handle malformed ext_pillar From 09f3fc698946a77c89085a168e6803ea4b099535 Mon Sep 17 00:00:00 2001 From: vernoncole Date: Wed, 27 Sep 2017 14:26:32 -0600 Subject: [PATCH 563/769] merge upstream develop --- src/saltext/consul/cache/__init__.py | 2 +- src/saltext/consul/cache/consul.py | 9 +- src/saltext/consul/pillar/__init__.py | 153 ++++++++++++++++----- src/saltext/consul/pillar/consul_pillar.py | 3 +- 4 files changed, 132 insertions(+), 35 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 94d7a36..fc5e5f0 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -73,7 +73,7 @@ def __init__(self, opts, cachedir=None, **kwargs): self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) else: self.cachedir = cachedir - self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS) + self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS['cache']) self.serial = Serial(opts) self._modules = None self._kwargs = kwargs diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index d8f1b32..d226cad 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -4,6 +4,8 @@ .. versionadded:: 2016.11.2 +:depends: python-consul >= 0.2.0 + It is up to the system administrator to set up and configure the Consul infrastructure. All is needed for this plugin is a working Consul agent with a read-write access to the key-value store. @@ -81,8 +83,11 @@ def __virtual__(): 'verify': __opts__.get('consul.verify', True), } - global api - api = consul.Consul(**consul_kwargs) + try: + global api + api = consul.Consul(**consul_kwargs) + except AttributeError: + return (False, "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed") return __virtualname__ diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f93f4ee..8a07562 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -13,6 +13,7 @@ import tornado.gen import sys import traceback +import inspect # Import salt libs import salt.loader @@ -23,6 +24,8 @@ import salt.utils.url import salt.utils.cache import salt.utils.crypt +import salt.utils.dictupdate +import salt.utils.args from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge @@ -36,7 +39,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -55,12 +58,14 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, + extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -72,15 +77,62 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None 'local': AsyncPillar, }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) -class AsyncRemotePillar(object): +class RemotePillarMixin(object): + ''' + Common remote pillar functionality + ''' + def get_ext_pillar_extra_minion_data(self, opts): + ''' + Returns the extra data from the minion's opts dict (the config file). + + This data will be passed to external pillar functions. + ''' + def get_subconfig(opts_key): + ''' + Returns a dict containing the opts key subtree, while maintaining + the opts structure + ''' + ret_dict = aux_dict = {} + config_val = opts + subkeys = opts_key.split(':') + # Build an empty dict with the opts path + for subkey in subkeys[:-1]: + aux_dict[subkey] = {} + aux_dict = aux_dict[subkey] + if not config_val.get(subkey): + # The subkey is not in the config + return {} + config_val = config_val[subkey] + if subkeys[-1] not in config_val: + return {} + aux_dict[subkeys[-1]] = config_val[subkeys[-1]] + return ret_dict + + extra_data = {} + if 'pass_to_ext_pillars' in opts: + if not isinstance(opts['pass_to_ext_pillars'], list): + log.exception('\'pass_to_ext_pillars\' config is malformed.') + raise SaltClientError('\'pass_to_ext_pillars\' config is ' + 'malformed.') + for key in opts['pass_to_ext_pillars']: + salt.utils.dictupdate.update(extra_data, + get_subconfig(key), + recursive_update=True, + merge_lists=True) + log.trace('ext_pillar_extra_data = {0}'.format(extra_data)) + return extra_data + + +class AsyncRemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -93,6 +145,14 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) @tornado.gen.coroutine def compile_pillar(self): @@ -104,6 +164,7 @@ def compile_pillar(self): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -126,12 +187,12 @@ def compile_pillar(self): raise tornado.gen.Return(ret_pillar) -class RemotePillar(object): +class RemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -144,6 +205,14 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) def compile_pillar(self): ''' @@ -154,6 +223,7 @@ def compile_pillar(self): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -187,7 +257,7 @@ class PillarCache(object): ''' # TODO ABC? def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -265,7 +335,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.minion_id = minion_id self.ext = ext if pillarenv is None: @@ -311,6 +381,10 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.pillar_override, dict): self.pillar_override = {} log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') def __valid_on_demand_ext_pillar(self, opts): ''' @@ -416,20 +490,19 @@ def get_tops(self): self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": @@ -768,17 +841,35 @@ def _external_pillar_data(self, pillar, val, key): Builds actual pillar data structure and updates the ``pillar`` variable ''' ext = None + args = salt.utils.args.get_function_argspec(self.ext_pillars[key]).args if isinstance(val, dict): - ext = self.ext_pillars[key](self.minion_id, pillar, **val) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, + extra_minion_data=self.extra_minion_data, **val) + else: + ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - ext = self.ext_pillars[key](self.minion_id, - pillar, - *val) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, *val, + extra_minion_data=self.extra_minion_data) + else: + ext = self.ext_pillars[key](self.minion_id, + pillar, + *val) else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - val) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, + pillar, + val, + extra_minion_data=self.extra_minion_data) + else: + ext = self.ext_pillars[key](self.minion_id, + pillar, + val) return ext def ext_pillar(self, pillar, errors=None): diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 0d10b80..d661649 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -167,7 +167,8 @@ def ext_pillar(minion_id, opts['target'] = match.group(1) temp = temp.replace(match.group(0), '') checker = salt.utils.minions.CkMinions(__opts__) - minions = checker.check_minions(opts['target'], 'compound') + _res = checker.check_minions(opts['target'], 'compound') + minions = _res['minions'] if minion_id not in minions: return {} From cd59900bf5cf0482bb4c264be22c21b3c8e0dc7e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 9 Oct 2017 12:53:31 -0500 Subject: [PATCH 564/769] Move 13 functions from salt.utils to salt.utils.data These functions are: - salt.utils.compare_dicts - salt.utils.compare_lists - salt.utils.decode_dict - salt.utils.decode_list - salt.utils.exactly_n - salt.utils.exactly_one - salt.utils.traverse_dict - salt.utils.filter_by - salt.utils.traverse_dict_and_list - salt.utils.subdict_match - salt.utils.substr_in_list - salt.utils.is_dictlist - salt.utils.repack_dictlist --- src/saltext/consul/pillar/__init__.py | 49 +++++++++++++++------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e6da1d8..0848d76 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -21,16 +21,20 @@ import salt.minion import salt.crypt import salt.transport -import salt.utils.url +import salt.utils.args import salt.utils.cache import salt.utils.crypt +import salt.utils.data import salt.utils.dictupdate -import salt.utils.args +import salt.utils.url from salt.exceptions import SaltClientError from salt.template import compile_template -from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict from salt.version import __version__ +# Even though dictupdate is imported, invoking salt.utils.dictupdate.merge here +# causes an UnboundLocalError. This should be investigated and fixed, but until +# then, leave the import directly below this comment intact. +from salt.utils.dictupdate import merge # Import 3rd-party libs from salt.ext import six @@ -905,11 +909,12 @@ def ext_pillar(self, pillar, errors=None): ext = None # Bring in CLI pillar data if self.pillar_override: - pillar = merge(pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge( + pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) for run in self.opts['ext_pillar']: if not isinstance(run, dict): @@ -961,11 +966,12 @@ def compile_pillar(self, ext=True): self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) - pillar = merge(self.opts['pillar'], - pillar, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge( + self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -988,11 +994,12 @@ def compile_pillar(self, ext=True): pillar['_errors'] = errors if self.pillar_override: - pillar = merge(pillar, - self.pillar_override, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + pillar = merge( + pillar, + self.pillar_override, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: @@ -1008,11 +1015,11 @@ def decrypt_pillar(self, pillar): decrypt_pillar = self.opts['decrypt_pillar'] if not isinstance(decrypt_pillar, dict): decrypt_pillar = \ - salt.utils.repack_dictlist(self.opts['decrypt_pillar']) + salt.utils.data.repack_dictlist(self.opts['decrypt_pillar']) if not decrypt_pillar: errors.append('decrypt_pillar config option is malformed') for key, rend in six.iteritems(decrypt_pillar): - ptr = salt.utils.traverse_dict( + ptr = salt.utils.data.traverse_dict( pillar, key, default=None, @@ -1044,7 +1051,7 @@ def decrypt_pillar(self, pillar): # parent is the pillar dict itself. ptr = pillar else: - ptr = salt.utils.traverse_dict( + ptr = salt.utils.data.traverse_dict( pillar, parent, default=None, From 43479c5b23dc5907faae75fe421a6cb78cbc6450 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 15 Oct 2017 14:59:17 -0500 Subject: [PATCH 565/769] Use one salt.utils.gitfs.GitFS instance per thread This reduces some of the overhead caused by many concurrent fileclient requests from minions. Additionally, initializing remotes has been folded into the GitBase class' dunder init. Instantiating an instance and initializing its remotes were initially split so that an object could simply be created with the master opts so that the configuration was loaded and nothing else, allowing for certain cases (like clearing the cache files) where we didn't need to actually initialize the remotes. But this both A) presents a problem when the instance being used is a singleton, as you don't want to be re-initializing the remotes all the time, and B) suppressing initialization can be (and is now being) done via a new argument to the dunder init. --- src/saltext/consul/pillar/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0848d76..37cb1bd 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -891,11 +891,11 @@ def ext_pillar(self, pillar, errors=None): # Avoid circular import import salt.utils.gitfs import salt.pillar.git_pillar - git_pillar = salt.utils.gitfs.GitPillar(self.opts) - git_pillar.init_remotes( + git_pillar = salt.utils.gitfs.GitPillar( + self.opts, self.ext['git'], - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY) git_pillar.fetch_remotes() except TypeError: # Handle malformed ext_pillar From 9361436389c8ac6c1ae4465c786309f16a2880ab Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:41:44 -0700 Subject: [PATCH 566/769] fix typo fix typo for clarity --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 68a4bd3..a6592cf 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1953,7 +1953,7 @@ def status_peers(consul_url): :param consul_url: The Consul server URL. :return: Retrieves the Raft peers for the - datacenter in which the the agent is running. + datacenter in which the agent is running. CLI Example: From fdeb37536914b7a197585814215f43394c9bcff8 Mon Sep 17 00:00:00 2001 From: Volodymyr Samodid Date: Tue, 7 Nov 2017 10:24:56 +0200 Subject: [PATCH 567/769] fix docstring errors during rendering --- src/saltext/consul/sdb/consul.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py index 16cce0e..531b8f8 100644 --- a/src/saltext/consul/sdb/consul.py +++ b/src/saltext/consul/sdb/consul.py @@ -13,6 +13,7 @@ requires very little. For example: .. code-block:: yaml + myconsul: driver: consul host: 127.0.0.1 From 24f4f1dffc28455264e5685f4e81bd5257929536 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 22:09:09 -0600 Subject: [PATCH 568/769] Rename 'environment' config option to 'saltenv' 'environment' predates the 'saltenv' nomenclature, and since we have been using saltenv on the CLI for years, and now also have pillarenv, it makes sense to make this change. --- src/saltext/consul/pillar/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 24c7c48..903d0f5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -138,7 +138,7 @@ class AsyncRemotePillar(RemotePillarMixin): def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts - self.opts['environment'] = saltenv + self.opts['saltenv'] = saltenv self.ext = ext self.grains = grains self.minion_id = minion_id @@ -165,7 +165,7 @@ def compile_pillar(self): ''' load = {'id': self.minion_id, 'grains': self.grains, - 'saltenv': self.opts['environment'], + 'saltenv': self.opts['saltenv'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, 'extra_minion_data': self.extra_minion_data, @@ -198,7 +198,7 @@ class RemotePillar(RemotePillarMixin): def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts - self.opts['environment'] = saltenv + self.opts['saltenv'] = saltenv self.ext = ext self.grains = grains self.minion_id = minion_id @@ -224,7 +224,7 @@ def compile_pillar(self): ''' load = {'id': self.minion_id, 'grains': self.grains, - 'saltenv': self.opts['environment'], + 'saltenv': self.opts['saltenv'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, 'extra_minion_data': self.extra_minion_data, @@ -445,9 +445,9 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): else: opts['grains'] = grains # Allow minion/CLI saltenv/pillarenv to take precedence over master - opts['environment'] = saltenv \ + opts['saltenv'] = saltenv \ if saltenv is not None \ - else opts.get('environment') + else opts.get('saltenv') opts['pillarenv'] = pillarenv \ if pillarenv is not None \ else opts.get('pillarenv') From 16933147ab182571e60126431253c7451c6f71e6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 7 Dec 2017 03:20:08 -0600 Subject: [PATCH 569/769] Roll back use of explicit unicode literals --- src/saltext/consul/__init__.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 21302c9..84e5d84 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,7 +7,6 @@ from __future__ import absolute_import import warnings -# future lint: disable=non-unicode-string # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( 'once', # Show once @@ -15,19 +14,18 @@ DeprecationWarning, # This filter is for DeprecationWarnings r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' ) -# future lint: enable=non-unicode-string # While we are supporting Python2.6, hide nested with-statements warnings warnings.filterwarnings( - u'ignore', - u'With-statements now directly support multiple context managers', + 'ignore', + 'With-statements now directly support multiple context managers', DeprecationWarning ) # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( - u'ignore', - u'^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + 'ignore', + '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', UserWarning ) @@ -39,7 +37,7 @@ def __define_global_system_encoding_variable__(): # and reset to None encoding = None - if not sys.platform.startswith(u'win') and sys.stdin is not None: + if not sys.platform.startswith('win') and sys.stdin is not None: # On linux we can rely on sys.stdin for the encoding since it # most commonly matches the filesystem encoding. This however # does not apply to windows @@ -65,16 +63,16 @@ def __define_global_system_encoding_variable__(): # the way back to ascii encoding = sys.getdefaultencoding() if not encoding: - if sys.platform.startswith(u'darwin'): + if sys.platform.startswith('darwin'): # Mac OS X uses UTF-8 - encoding = u'utf-8' - elif sys.platform.startswith(u'win'): + encoding = 'utf-8' + elif sys.platform.startswith('win'): # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” # to refer to whatever the currently configured encoding is. - encoding = u'mbcs' + encoding = 'mbcs' else: # On linux default to ascii as a last resort - encoding = u'ascii' + encoding = 'ascii' # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: @@ -85,7 +83,7 @@ def __define_global_system_encoding_variable__(): import builtins # pylint: disable=import-error # Define the detected encoding as a built-in variable for ease of use - setattr(builtins, u'__salt_system_encoding__', encoding) + setattr(builtins, '__salt_system_encoding__', encoding) # This is now garbage collectable del sys From 3c13bec82f36e0ad77cfb5d1ddde3b6b7e0c1deb Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 14 Dec 2017 00:26:39 -0600 Subject: [PATCH 570/769] [PY3] Add unicode_literals to files in root salt/ dir (and associated tests) --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 84e5d84..21e721c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -4,7 +4,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import warnings # All salt related deprecation warnings should be shown once each! From 6d391579c49d29616b712ffd0246709816fc3692 Mon Sep 17 00:00:00 2001 From: s8weber Date: Thu, 14 Dec 2017 19:08:30 -0500 Subject: [PATCH 571/769] fix pillar includes from merging over the current sls defines ihttps://github.com/saltstack/salt/issues/39860 --- src/saltext/consul/pillar/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 8d5eb7e..cd0e69f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -664,6 +664,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: + # render included state(s) + include_states = [] for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) @@ -685,16 +687,23 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate = { key_fragment: nstate } - + include_states.append(nstate) + if err: + errors += err + if include_states: + # merge included state(s) with the current state merged last + include_states.append(state) + state = None + for s in include_states: + if state is None: + state = s + else: state = merge( state, - nstate, + s, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) - - if err: - errors += err return state, mods, errors def render_pillar(self, matches, errors=None): From 70660ae23244436ff520258fa91157c060e98ee9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 15 Dec 2017 12:14:18 -0600 Subject: [PATCH 572/769] [PY3] Add print_function import to files with unicode_literals already added This makes the 2.x usage invalid syntax and forces the use of print as a function. This adds the import to the files which I've updated in the last couple of days but forgot to add it. --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 21e721c..22f8145 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -4,7 +4,7 @@ ''' # Import Python libs -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import warnings # All salt related deprecation warnings should be shown once each! From 7793cd7ab33632c0343f88d293b7c5cbb443e432 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 22 Dec 2017 13:54:53 -0600 Subject: [PATCH 573/769] Replace json module usage with a helper to ensure unicode content is handled properly This adds wrappers for json.dump{,s} which disable `ensure_ascii` by default. --- src/saltext/consul/modules/consul.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 15ccba9..f8a8158 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -8,6 +8,12 @@ # Import Python Libs from __future__ import absolute_import +import base64 +import logging + +# Import salt libs +import salt.utils.http +import salt.utils.json # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin @@ -16,13 +22,6 @@ import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module -# Import salt libs -import salt.utils.http - -import base64 -import json - -import logging log = logging.getLogger(__name__) from salt.exceptions import SaltInvocationError @@ -84,7 +83,7 @@ def _query(function, if data is None: data = {} - data = json.dumps(data) + data = salt.utils.json.dumps(data) result = salt.utils.http.query( url, From 19458ed953c70b6c2f90cf5c85f8838071a93ba6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 28 Dec 2017 13:18:00 -0600 Subject: [PATCH 574/769] [PY3] Add unicode_literals to sdb and serializers NOTE: the other serializers were done in a previous commit. --- src/saltext/consul/sdb/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py index 531b8f8..567e299 100644 --- a/src/saltext/consul/sdb/consul.py +++ b/src/saltext/consul/sdb/consul.py @@ -27,7 +27,7 @@ The ``driver`` refers to the Consul module, all other options are optional. For option details see: https://python-consul.readthedocs.io/en/latest/#consul ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals from salt.exceptions import CommandExecutionError From 353cfee211107f99fb33df9d40528cfcf11070ce Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 27 Dec 2017 22:31:50 -0600 Subject: [PATCH 575/769] Replace yaml usage with a helper to ensure unicode is handled properly Without allow_unicode=True, unicode characters are processed through the str representer and on Python 2 are dumped as a Unicode code point (i.e. a literal \u0414). This commit makes allow_unicode=True the default in our salt.utils.yamlloader.safe_dump() helper. It also adds a new salt.utils.yamlloader.dump() helper which wraps yaml.dump() and also makes allow_unicode=True the default. To make importing and using our custom yaml loader/dumper easier, a convenience module called salt.utils.yaml has been added, which does a wildcard import from both salt.utils.yamldumper and salt.utils.yamlloader. Refs to yaml.load/dump and yaml.safe_load/safe_dump have been updated to salt.utils.yaml, to ensure that unicode is handled properly. --- src/saltext/consul/pillar/consul_pillar.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index d661649..9e0369d 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -125,12 +125,11 @@ # Import python libs import logging import re -import yaml from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge import salt.utils.minions -from salt.utils.yamlloader import SaltYamlSafeLoader +import salt.utils.yaml # Import third party libs try: @@ -253,10 +252,7 @@ def pillar_format(ret, keys, value): # If value is not None then it's a string # Use YAML to parse the data # YAML strips whitespaces unless they're surrounded by quotes - pillar_value = yaml.load( - value, - Loader=SaltYamlSafeLoader - ) + pillar_value = salt.utils.yaml.safe_load(value) keyvalue = keys.pop() pil = {keyvalue: pillar_value} From f50c304ad9b14ca9e3c1bda889b1c8568ac94e22 Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Thu, 4 Jan 2018 22:17:31 +0800 Subject: [PATCH 576/769] Fix #22063: pillar wildcard support include --- src/saltext/consul/pillar/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 903d0f5..daae0ca 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -753,6 +753,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: + matched_pstates = [] for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) @@ -760,6 +761,16 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key = v.get('key', None) else: key = None + + try: + matched_pstates = fnmatch.filter(self.avail[saltenv], sub_sls) + except KeyError: + errors.extend( + ['No matching pillar environment for environment ' + '\'{0}\' found'.format(saltenv)] + ) + + for sub_sls in matched_pstates: if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, From fb8c6962b9744fcb1b954c41cd163ff4808a76da Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 3 Jan 2018 17:19:53 -0600 Subject: [PATCH 577/769] [PY3] Add unicode_literals to ext_pillar modules --- src/saltext/consul/pillar/__init__.py | 48 ++++++++++------------ src/saltext/consul/pillar/consul_pillar.py | 8 ++-- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 903d0f5..62b56c4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -4,7 +4,7 @@ ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import copy import fnmatch import os @@ -58,7 +58,7 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, log.debug('Determining pillar cache') if opts['pillar_cache']: log.info('Compiling pillar from cache') - log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) + log.debug('get_pillar using pillar cache with ext: %s', ext) return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, @@ -127,7 +127,7 @@ def get_subconfig(opts_key): get_subconfig(key), recursive_update=True, merge_lists=True) - log.trace('ext_pillar_extra_data = {0}'.format(extra_data)) + log.trace('ext_pillar_extra_data = %s', extra_data) return extra_data @@ -238,8 +238,8 @@ def compile_pillar(self): if not isinstance(ret_pillar, dict): log.error( - 'Got a bad pillar from master, type {0}, expecting dict: ' - '{1}'.format(type(ret_pillar).__name__, ret_pillar) + 'Got a bad pillar from master, type %s, expecting dict: %s', + type(ret_pillar).__name__, ret_pillar ) return {} return ret_pillar @@ -298,7 +298,7 @@ def fetch_pillar(self): In the event of a cache miss, we need to incur the overhead of caching a new pillar. ''' - log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) + log.debug('Pillar cache getting external pillar with ext: %s', self.ext) fresh_pillar = Pillar(self.opts, self.grains, self.minion_id, @@ -310,27 +310,27 @@ def fetch_pillar(self): return fresh_pillar.compile_pillar() def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs - log.debug('Scanning pillar cache for information about minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) - log.debug('Scanning cache: {0}'.format(self.cache._dict)) + log.debug('Scanning pillar cache for information about minion %s and pillarenv %s', self.minion_id, self.pillarenv) + log.debug('Scanning cache: %s', self.cache._dict) # Check the cache! if self.minion_id in self.cache: # Keyed by minion_id # TODO Compare grains, etc? if self.pillarenv in self.cache[self.minion_id]: # We have a cache hit! Send it back. - log.debug('Pillar cache hit for minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) + log.debug('Pillar cache hit for minion %s and pillarenv %s', self.minion_id, self.pillarenv) return self.cache[self.minion_id][self.pillarenv] else: # We found the minion but not the env. Store it. fresh_pillar = self.fetch_pillar() self.cache[self.minion_id][self.pillarenv] = fresh_pillar - log.debug('Pillar cache miss for pillarenv {0} for minion {1}'.format(self.pillarenv, self.minion_id)) + log.debug('Pillar cache miss for pillarenv %s for minion %s', self.pillarenv, self.minion_id) return fresh_pillar else: # We haven't seen this minion yet in the cache. Store it. fresh_pillar = self.fetch_pillar() self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} - log.debug('Pillar cache miss for minion {0}'.format(self.minion_id)) - log.debug('Current pillar cache: {0}'.format(self.cache._dict)) # FIXME hack! + log.debug('Pillar cache miss for minion %s', self.minion_id) + log.debug('Current pillar cache: %s', self.cache._dict) # FIXME hack! return fresh_pillar @@ -534,8 +534,7 @@ def get_tops(self): errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) - log.error('Pillar rendering failed for minion {0}: '.format(self.minion_id), - exc_info=True) + log.exception('Pillar rendering failed for minion %s', self.minion_id) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): @@ -686,8 +685,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): fn_ = self.client.get_state(sls, saltenv).get('dest', False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): - log.debug('Skipping ignored and missing SLS \'{0}\' in' - ' environment \'{1}\''.format(sls, saltenv)) + log.debug('Skipping ignored and missing SLS \'%s\' in ' + 'environment \'%s\'', sls, saltenv) return None, mods, errors elif self.opts['pillar_roots'].get(saltenv): msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not' @@ -820,15 +819,12 @@ def render_pillar(self, matches, errors=None): if pstate is not None: if not isinstance(pstate, dict): log.error( - 'The rendered pillar sls file, \'{0}\' state did ' + 'The rendered pillar sls file, \'%s\' state did ' 'not return the expected data format. This is ' 'a sign of a malformed pillar sls file. Returned ' - 'errors: {1}'.format( - sls, - ', '.join( - ['\'{0}\''.format(e) for e in errors] - ) - ) + 'errors: %s', + sls, + ', '.join(["'{0}'".format(e) for e in errors]) ) continue pillar = merge( @@ -926,8 +922,8 @@ def ext_pillar(self, pillar, errors=None): for key, val in six.iteritems(run): if key not in self.ext_pillars: log.critical( - 'Specified ext_pillar interface {0} is ' - 'unavailable'.format(key) + 'Specified ext_pillar interface %s is unavailable', + key ) continue try: @@ -990,7 +986,7 @@ def compile_pillar(self, ext=True): pillar['master'] = mopts if errors: for error in errors: - log.critical('Pillar render error: {0}'.format(error)) + log.critical('Pillar render error: %s', error) pillar['_errors'] = errors if self.pillar_override: diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 9e0369d..d4ff2a4 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -120,7 +120,7 @@ - consul: my_consul_config root=salt target="L@salt.example.com and G@osarch:x86_64" ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging @@ -224,7 +224,7 @@ def fetch_tree(client, path): ret = {} has_children = re.compile(r'/$') - log.debug('Fetched items: %r', format(items)) + log.debug('Fetched items: %r', items) if items is None: return ret @@ -232,10 +232,10 @@ def fetch_tree(client, path): key = re.sub(r'^' + path + '/?', '', item['Key']) if key != '': log.debug('key/path - %s: %s', path, key) - log.debug('has_children? %r', format(has_children.search(key))) + log.debug('has_children? %r', has_children.search(key)) if has_children.search(key) is None: ret = pillar_format(ret, key.split('/'), item['Value']) - log.debug('Fetching subkeys for key: %r', format(item)) + log.debug('Fetching subkeys for key: %r', item) return ret From 172e4d30e3bba9cb4bc103dcec7ffa7202f9feeb Mon Sep 17 00:00:00 2001 From: s8weber Date: Thu, 14 Dec 2017 19:08:30 -0500 Subject: [PATCH 578/769] fix pillar includes from merging over the current sls defines ihttps://github.com/saltstack/salt/issues/39860 --- src/saltext/consul/pillar/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c6edcc4..13a0bb8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -664,6 +664,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): log.error(msg) errors.append(msg) else: + # render included state(s) + include_states = [] for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) @@ -685,16 +687,23 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate = { key_fragment: nstate } - + include_states.append(nstate) + if err: + errors += err + if include_states: + # merge included state(s) with the current state merged last + include_states.append(state) + state = None + for s in include_states: + if state is None: + state = s + else: state = merge( state, - nstate, + s, self.merge_strategy, self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) - - if err: - errors += err return state, mods, errors def render_pillar(self, matches, errors=None): From 3516b88abafaaeb2abd31f2e1624a520ffe9ac9a Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 5 Jan 2018 11:13:46 -0600 Subject: [PATCH 579/769] [PY3] Add unicode_literals to cache and cli modules --- src/saltext/consul/cache/__init__.py | 10 +++++----- src/saltext/consul/cache/consul.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index fc5e5f0..68ffcdc 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -6,7 +6,7 @@ ''' # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging import time @@ -323,10 +323,10 @@ def fetch(self, bank, key): if record is not None and record[0] + self.expire >= now: if self.debug: self.hit += 1 - log.debug('MemCache stats (call/hit/rate): ' - '{0}/{1}/{2}'.format(self.call, - self.hit, - float(self.hit) / self.call)) + log.debug( + 'MemCache stats (call/hit/rate): %s/%s/%s', + self.call, self.hit, float(self.hit) / self.call + ) # update atime and return record[0] = now self.storage[(bank, key)] = record diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index d226cad..2f8e878 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -46,7 +46,7 @@ .. _`python-consul documentation`: https://python-consul.readthedocs.io/en/latest/#consul ''' -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging try: import consul From 818b0b2a094ba587a8b201bf8646689befd70c59 Mon Sep 17 00:00:00 2001 From: Marlowe W Date: Mon, 15 Jan 2018 12:50:28 +0800 Subject: [PATCH 580/769] split pillar include unit test --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0f20fa0..a70f7f9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -765,14 +765,14 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key = None try: - matched_pstates = fnmatch.filter(self.avail[saltenv], sub_sls) + matched_pstates += fnmatch.filter(self.avail[saltenv], sub_sls) except KeyError: errors.extend( ['No matching pillar environment for environment ' '\'{0}\' found'.format(saltenv)] ) - for sub_sls in matched_pstates: + for sub_sls in set(matched_pstates): if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, From dbb11a2f2f0681e5b4ad55ae2fac831b8fe0230d Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Fri, 19 Jan 2018 21:48:35 +0300 Subject: [PATCH 581/769] Salt Unicode Update --- src/saltext/consul/modules/consul.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index f8a8158..067c26e 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -7,7 +7,7 @@ ''' # Import Python Libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import base64 import logging @@ -1254,7 +1254,7 @@ def session_create(consul_url=None, token=None, **kwargs): if 'ttl' in kwargs: _ttl = kwargs['ttl'] - if str(_ttl).endswith('s'): + if six.text_type(_ttl).endswith('s'): _ttl = _ttl[:-1] if int(_ttl) < 0 or int(_ttl) > 3600: From 8ef64629296dfcec8ca5336e478e190f46f9e119 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko Date: Mon, 22 Jan 2018 02:06:05 +0300 Subject: [PATCH 582/769] Fixed lint errors --- src/saltext/consul/modules/consul.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 067c26e..8c4a611 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -16,11 +16,8 @@ import salt.utils.json # Import 3rd-party libs -# pylint: disable=import-error,no-name-in-module,redefined-builtin -from salt.ext.six.moves.urllib.parse import urljoin as _urljoin -import salt.ext.six -import salt.ext.six.moves.http_client -# pylint: enable=import-error,no-name-in-module +from salt.ext import six +from salt.ext.six.moves import http_client, urllib log = logging.getLogger(__name__) @@ -78,8 +75,8 @@ def _query(function, token = _get_token() headers = {"X-Consul-Token": token, "Content-Type": "application/json"} - base_url = _urljoin(consul_url, '{0}/'.format(api_version)) - url = _urljoin(base_url, function, False) + base_url = urllib.parse.urljoin(consul_url, '{0}/'.format(api_version)) + url = urllib.parse.urljoin(base_url, function, False) if data is None: data = {} @@ -96,12 +93,12 @@ def _query(function, opts=__opts__, ) - if result.get('status', None) == salt.ext.six.moves.http_client.OK: + if result.get('status', None) == http_client.OK: ret['data'] = result.get('dict', result) ret['res'] = True - elif result.get('status', None) == salt.ext.six.moves.http_client.NO_CONTENT: + elif result.get('status', None) == http_client.NO_CONTENT: ret['res'] = False - elif result.get('status', None) == salt.ext.six.moves.http_client.NOT_FOUND: + elif result.get('status', None) == http_client.NOT_FOUND: ret['data'] = 'Key not found.' ret['res'] = False else: @@ -1015,7 +1012,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): return ret lc_kwargs = dict() - for k, v in salt.ext.six.iteritems(kwargs): + for k, v in six.iteritems(kwargs): lc_kwargs[k.lower()] = v if 'name' in lc_kwargs: @@ -1043,7 +1040,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): if 'check' in lc_kwargs: dd = dict() - for k, v in salt.ext.six.iteritems(lc_kwargs['check']): + for k, v in six.iteritems(lc_kwargs['check']): dd[k.lower()] = v interval_required = False check_dd = dict() @@ -1068,7 +1065,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): if 'Interval' in check_dd: del check_dd['Interval'] # not required, so ignore it - if len(check_dd) > 0: + if check_dd > 0: data['Check'] = check_dd # if empty, ignore it function = 'agent/service/register' From 84a8d1e003c2a89053b83d04180d8422f80cbd20 Mon Sep 17 00:00:00 2001 From: campbell Date: Sun, 11 Feb 2018 16:18:15 +0000 Subject: [PATCH 583/769] stop 'data' being assigned with serialised empty dict for GET requests --- src/saltext/consul/modules/consul.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 8c4a611..c07d9cf 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -78,9 +78,12 @@ def _query(function, base_url = urllib.parse.urljoin(consul_url, '{0}/'.format(api_version)) url = urllib.parse.urljoin(base_url, function, False) - if data is None: - data = {} - data = salt.utils.json.dumps(data) + if method == 'GET': + data = None + else: + if data is None: + data = {} + data = salt.utils.json.dumps(data) result = salt.utils.http.query( url, From feb756eb6c45dee98b06f45b59bfd558f451de17 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 28 Feb 2018 13:07:11 -0600 Subject: [PATCH 584/769] Normalize global git_pillar/winrepo config items We were only normalizing the global version of a config parameter if it had a per-remote counterpart. This was all well and good for gitfs, which doesn't have any config values that are global-only, but git_pillar and winrepo have config params which are global-only, and they were therefore not being normalized. This fixes that oversight, and also makes a few other changes in salt.utils.gitfs to ensure that we're not pulling from the un-normalized values from the opts dict, when we need to fall back to global defaults. Additionally, it fixes an issue in which the `git_pillar_branch` config item was overriding the branch set in a git_pillar remote. --- src/saltext/consul/pillar/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 13a0bb8..1c43c4e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -792,7 +792,8 @@ def ext_pillar(self, pillar, pillar_dirs, errors=None): git_pillar.init_remotes( self.ext['git'], salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + salt.pillar.git_pillar.PER_REMOTE_ONLY, + salt.pillar.git_pillar.GLOBAL_ONLY) git_pillar.fetch_remotes() except TypeError: # Handle malformed ext_pillar From fcb27615ad9ff4c73498cc2d01edcc864c1cc1aa Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Thu, 1 Mar 2018 17:11:45 +0100 Subject: [PATCH 585/769] Refactor Pillar.get_tops to remove code duplication self.client.cache_file() and compile_template() are used in both if branches. Reduce this code duplication. --- src/saltext/consul/pillar/__init__.py | 45 ++++++++++----------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a70f7f9..02be328 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -484,6 +484,7 @@ def get_tops(self): errors = [] # Gather initial top files try: + saltenvs = set() if self.opts['pillarenv']: # If the specified pillarenv is not present in the available # pillar environments, do not cache the pillar top file. @@ -494,19 +495,7 @@ def get_tops(self): self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: - top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) - if top: - tops[self.opts['pillarenv']] = [ - compile_template( - top, - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + saltenvs.add(self.opts['pillarenv']) else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": @@ -514,22 +503,20 @@ def get_tops(self): continue if not self.saltenv and not saltenv == 'base': continue - top = self.client.cache_file( - self.opts['state_top'], - saltenv - ) - if top: - tops[saltenv].append( - compile_template( - top, - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - saltenv=saltenv, - _pillar_rend=True, - ) - ) + saltenvs.add(saltenv) + + for saltenv in saltenvs: + top = self.client.cache_file(self.opts['state_top'], saltenv) + if top: + tops[saltenv].append(compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + saltenv=saltenv, + _pillar_rend=True, + )) except Exception as exc: errors.append( ('Rendering Primary Top file failed, render error:\n{0}' From e63493e87f2b80483dee35fa4a9b7a0c75473fac Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Thu, 1 Mar 2018 17:46:52 +0100 Subject: [PATCH 586/769] Simplify saltenv calculation in Pillar.get_tops Pillar._get_envs returns a set of existing pillar environments. Instead of iterating over all environments and check condition for each environment, do the if conditions first and reduce the set based on the conditions. --- src/saltext/consul/pillar/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 02be328..95830cc 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -497,13 +497,9 @@ def get_tops(self): else: saltenvs.add(self.opts['pillarenv']) else: - for saltenv in self._get_envs(): - if self.opts.get('pillar_source_merging_strategy', None) == "none": - if self.saltenv and saltenv != self.saltenv: - continue - if not self.saltenv and not saltenv == 'base': - continue - saltenvs.add(saltenv) + saltenvs = self._get_envs() + if self.opts.get('pillar_source_merging_strategy', None) == "none": + saltenvs &= set([self.saltenv or 'base']) for saltenv in saltenvs: top = self.client.cache_file(self.opts['state_top'], saltenv) From 6f4562492f9788752725cf972c1aec0252caa35b Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 2 Mar 2018 20:25:28 +0100 Subject: [PATCH 587/769] Simplify envs/_get_envs methods Use list/set operations instead of iterating over the self.opts['file_roots'] keys individually. --- src/saltext/consul/pillar/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 95830cc..b3e8bb5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -469,10 +469,7 @@ def _get_envs(self): ''' Pull the file server environments out of the master options ''' - envs = set(['base']) - if 'file_roots' in self.opts: - envs.update(list(self.opts['file_roots'])) - return envs + return set(['base']) | set(self.opts.get('file_roots', [])) def get_tops(self): ''' From 7f069da5ef1bdc6b1a243b4eed1fed6d56f09367 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 2 Mar 2018 11:14:35 +0100 Subject: [PATCH 588/769] Support dynamic pillar_root environment Allow users to specify a __env__ pillar_root directory that applies equally to all environments that are not explicitly specified. fixes #20581 Signed-off-by: Benjamin Drung --- src/saltext/consul/pillar/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 1c43c4e..7741661 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -374,6 +374,15 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): opts['ext_pillar'].append(self.ext) else: opts['ext_pillar'] = [self.ext] + if '__env__' in opts['file_roots']: + env = opts.get('pillarenv') or opts.get('saltenv') or 'base' + if env not in opts['file_roots']: + log.debug("pillar environment '%s' maps to __env__ pillar_roots directory", env) + opts['file_roots'][env] = opts['file_roots'].pop('__env__') + else: + log.debug("pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", + env) + opts['file_roots'].pop('__env__') return opts def _get_envs(self): From fe69b68d8f81128587d71aa8bcd8733e02326b55 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 22 Mar 2018 13:32:13 -0500 Subject: [PATCH 589/769] Add option to return to pre-2017.7.3 pillar include merge order https://github.com/saltstack/salt/pull/45025 changed the pillar include merge order. This adds a new config option which allows a user to revert to the pre-2017.7.3 behavior. --- src/saltext/consul/pillar/__init__.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7741661..63bc758 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -696,11 +696,22 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate = { key_fragment: nstate } - include_states.append(nstate) + if not self.opts.get('pillar_includes_override_sls', False): + include_states.append(nstate) + else: + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) if err: errors += err - if include_states: - # merge included state(s) with the current state merged last + + if not self.opts.get('pillar_includes_override_sls', False): + # merge included state(s) with the current state + # merged last to ensure that its values are + # authoritative. include_states.append(state) state = None for s in include_states: From e7e91ce89033374685e821ea1c5452f26d6a80c1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 9 Apr 2018 11:33:56 -0500 Subject: [PATCH 590/769] Include exc_info in pillar render errors to aid in troubleshooting --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 25bf267..fc1e34f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -738,7 +738,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( sls, exc ) - log.critical(msg) + log.critical(msg, exc_info=True) if self.opts.get('pillar_safe_render_error', True): errors.append( 'Rendering SLS \'{0}\' failed. Please see master log for ' From 7909b4e6b1f22c787f7f84ea8d4c1375f3fc25a7 Mon Sep 17 00:00:00 2001 From: Campbell Date: Mon, 16 Apr 2018 14:24:39 +0100 Subject: [PATCH 591/769] Update consul.py Added param comment regarding ignoring data for GET requests in _query. --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index c07d9cf..8176b43 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -61,7 +61,7 @@ def _query(function, :param api_version The Consul api version :param function: The Consul api function to perform. :param method: The HTTP method, e.g. GET or POST. - :param data: The data to be sent for POST method. + :param data: The data to be sent for POST method. This param is ignored for GET requests. :return: The json response from the API call or False. ''' From 45cae5ab755ec96b3ad8fbd1a358f8c80f7c0726 Mon Sep 17 00:00:00 2001 From: Matei Albu Date: Sun, 6 May 2018 21:15:58 +0200 Subject: [PATCH 592/769] Option to merge current pillar with opts['pillar'] during pillar compile Fixes #47501 --- src/saltext/consul/pillar/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fc1e34f..fc3ce0a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1014,6 +1014,13 @@ def compile_pillar(self, ext=True): mopts['file_roots'] = self.actual_file_roots mopts['saltversion'] = __version__ pillar['master'] = mopts + if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False): + pillar = merge( + self.opts['pillar'], + pillar, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) if errors: for error in errors: log.critical('Pillar render error: %s', error) From d5bd0c5c4fd06421e7b29835504ba7bd779270c1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 28 May 2018 16:13:12 -0500 Subject: [PATCH 593/769] Fix all Sphinx warnings Well, all but one, which we expect to see --- src/saltext/consul/modules/consul.py | 45 +++++++++------------------- src/saltext/consul/sdb/consul.py | 1 + 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 33d746c..48a9997 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -114,7 +114,6 @@ def list_(consul_url=None, key=None, **kwargs): .. code-block:: bash salt '*' consul.list - salt '*' consul.list key='web' ''' @@ -164,17 +163,15 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): .. code-block:: bash salt '*' consul.get key='web/key1' - - salt '*' consul.list key='web' recurse='True - - salt '*' consul.list key='web' recurse='True' decode='True' + salt '*' consul.get key='web' recurse=True + salt '*' consul.get key='web' recurse=True decode=True By default values stored in Consul are base64 encoded, passing the decode option will show them as the decoded values. .. code-block:: bash - salt '*' consul.list key='web' recurse='True' decode='True' raw='True' + salt '*' consul.get key='web' recurse=True decode=True raw=True By default Consult will return other information about the key, the raw option will return only the raw value. @@ -233,11 +230,9 @@ def put(consul_url=None, key=None, value=None, **kwargs): salt '*' consul.put key='web/key1' value="Hello there" - salt '*' consul.put key='web/key1' value="Hello there" - acquire='d5d371f4-c380-5280-12fd-8810be175592' + salt '*' consul.put key='web/key1' value="Hello there" acquire='d5d371f4-c380-5280-12fd-8810be175592' - salt '*' consul.put key='web/key1' value="Hello there" - release='d5d371f4-c380-5280-12fd-8810be175592' + salt '*' consul.put key='web/key1' value="Hello there" release='d5d371f4-c380-5280-12fd-8810be175592' ''' ret = {} @@ -347,7 +342,6 @@ def delete(consul_url=None, key=None, **kwargs): .. code-block:: bash salt '*' consul.delete key='web' - salt '*' consul.delete key='web' recurse='True' ''' @@ -687,8 +681,7 @@ def agent_check_register(consul_url=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_check_register name='Memory Utilization' - script='/usr/local/bin/check_mem.py' interval='15s' + salt '*' consul.agent_check_register name='Memory Utilization' script='/usr/local/bin/check_mem.py' interval='15s' ''' ret = {} @@ -802,8 +795,7 @@ def agent_check_pass(consul_url=None, checkid=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_check_pass checkid='redis_check1' - note='Forcing check into passing state.' + salt '*' consul.agent_check_pass checkid='redis_check1' note='Forcing check into passing state.' ''' ret = {} @@ -851,8 +843,7 @@ def agent_check_warn(consul_url=None, checkid=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_check_warn checkid='redis_check1' - note='Forcing check into warning state.' + salt '*' consul.agent_check_warn checkid='redis_check1' note='Forcing check into warning state.' ''' ret = {} @@ -900,8 +891,7 @@ def agent_check_fail(consul_url=None, checkid=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_check_fail checkid='redis_check1' - note='Forcing check into critical state.' + salt '*' consul.agent_check_fail checkid='redis_check1' note='Forcing check into critical state.' ''' ret = {} @@ -963,9 +953,7 @@ def agent_service_register(consul_url=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_service_register name='redis' - tags='["master", "v1"]' address="127.0.0.1" port="8080" - check_script="/usr/local/bin/check_redis.py" interval="10s" + salt '*' consul.agent_service_register name='redis' tags='["master", "v1"]' address="127.0.0.1" port="8080" check_script="/usr/local/bin/check_redis.py" interval="10s" ''' ret = {} @@ -1106,8 +1094,7 @@ def agent_service_maintenance(consul_url=None, serviceid=None, **kwargs): .. code-block:: bash - salt '*' consul.agent_service_deregister serviceid='redis' - enable='True' reason='Down for upgrade' + salt '*' consul.agent_service_deregister serviceid='redis' enable='True' reason='Down for upgrade' ''' ret = {} @@ -1177,8 +1164,7 @@ def session_create(consul_url=None, **kwargs): .. code-block:: bash - salt '*' consul.session_create node='node1' name='my-session' - behavior='delete' ttl='3600s' + salt '*' consul.session_create node='node1' name='my-session' behavior='delete' ttl='3600s' ''' ret = {} @@ -1401,9 +1387,7 @@ def catalog_register(consul_url=None, **kwargs): .. code-block:: bash - salt '*' consul.catalog_register node='node1' address='192.168.1.1' - service='redis' service_address='127.0.0.1' service_port='8080' - service_id='redis_server1' + salt '*' consul.catalog_register node='node1' address='192.168.1.1' service='redis' service_address='127.0.0.1' service_port='8080' service_id='redis_server1' ''' ret = {} @@ -1504,8 +1488,7 @@ def catalog_deregister(consul_url=None, **kwargs): .. code-block:: bash - salt '*' consul.catalog_register node='node1' - serviceid='redis_server1' checkid='redis_check1' + salt '*' consul.catalog_register node='node1' serviceid='redis_server1' checkid='redis_check1' ''' ret = {} diff --git a/src/saltext/consul/sdb/consul.py b/src/saltext/consul/sdb/consul.py index 16cce0e..531b8f8 100644 --- a/src/saltext/consul/sdb/consul.py +++ b/src/saltext/consul/sdb/consul.py @@ -13,6 +13,7 @@ requires very little. For example: .. code-block:: yaml + myconsul: driver: consul host: 127.0.0.1 From 25ebb4a72b878ed3bac002da9c01b9140d4277f5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 6 Jun 2018 00:49:37 -0500 Subject: [PATCH 594/769] Clean up Sphinx warnings --- docs/ref/cache/all/salt.cache.consul.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst index 516a3b8..d28bcc0 100644 --- a/docs/ref/cache/all/salt.cache.consul.rst +++ b/docs/ref/cache/all/salt.cache.consul.rst @@ -1,5 +1,5 @@ -salt.cache.consul module -======================== +salt.cache.consul +================= .. automodule:: salt.cache.consul :members: From 9161c8e1b361eebdb6b746b75e3490d1911ebae5 Mon Sep 17 00:00:00 2001 From: Max Arnold Date: Thu, 21 Jun 2018 14:22:44 +0700 Subject: [PATCH 595/769] Fix quite common Exception typos --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6f09871..874dd85 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -960,7 +960,7 @@ def ext_pillar(self, pillar, errors=None): ) ) log.error( - 'Execption caught loading ext_pillar \'%s\':\n%s', + 'Exception caught loading ext_pillar \'%s\':\n%s', key, ''.join(traceback.format_tb(sys.exc_info()[2])) ) if ext: From e2be7d4e7f6af68495b3a7b118ff0081a6b9ac1c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 26 Jun 2018 15:04:58 +0200 Subject: [PATCH 596/769] Add backward-compatibility for older python-consul package dependencies --- src/saltext/consul/pillar/consul_pillar.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index d4ff2a4..bf5de52 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -134,10 +134,10 @@ # Import third party libs try: import consul - HAS_CONSUL = True - CONSUL_VERSION = consul.__version__ + if not hasattr(consul, '__version__'): + consul.__version__ = '0.1' # Some packages has no version, and so this pillar crashes on access to it. except ImportError: - HAS_CONSUL = False + consul = None __virtualname__ = 'consul' @@ -149,7 +149,7 @@ def __virtual__(): ''' Only return if python-consul is installed ''' - return __virtualname__ if HAS_CONSUL else False + return __virtualname__ if consul is not None else False def ext_pillar(minion_id, @@ -290,9 +290,9 @@ def get_conn(opts, profile): pillarenv = opts_merged.get('pillarenv') or 'base' params['dc'] = _resolve_datacenter(params['dc'], pillarenv) - if HAS_CONSUL: + if consul: # Sanity check. ACL Tokens are supported on python-consul 0.4.7 onwards only. - if CONSUL_VERSION < '0.4.7' and params.get('target'): + if consul.__version__ < '0.4.7' and params.get('target'): params.pop('target') return consul.Consul(**params) else: From 314c377647c42aa19f25f1f4471ff3230a595648 Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Fri, 29 Jun 2018 16:20:22 -0700 Subject: [PATCH 597/769] Added to consul pillar the ability to nest consul data at a subkey in pillar and to read non-YAML data. --- src/saltext/consul/pillar/consul_pillar.py | 68 ++++++++++++++++++++-- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index bf5de52..4b6df47 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -119,6 +119,21 @@ ext_pillar: - consul: my_consul_config root=salt target="L@salt.example.com and G@osarch:x86_64" +The data from Consul can be merged into a nested key in Pillar. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config pillar_root=consul_data + +By default, keys containing YAML data will be deserialized before being merged into Pillar. +This behavior can be disabled by setting ``parse_yaml`` to ``false``. + +.. code-block:: yaml + + ext_pillar: + - consul: my_consul_config parse_yaml=false + ''' from __future__ import absolute_import, print_function, unicode_literals @@ -168,6 +183,7 @@ def ext_pillar(minion_id, checker = salt.utils.minions.CkMinions(__opts__) _res = checker.check_minions(opts['target'], 'compound') minions = _res['minions'] + log.debug('Targeted minions: %r', minions) if minion_id not in minions: return {} @@ -179,6 +195,14 @@ def ext_pillar(minion_id, else: opts['root'] = "" + pillar_root_re = re.compile('pillar_root=(\S*)') # pylint: disable=W1401 + match = pillar_root_re.search(temp) + if match: + opts['pillar_root'] = match.group(1) + temp = temp.replace(match.group(0), '') + else: + opts['pillar_root'] = "" + profile_re = re.compile('(?:profile=)?(\S+)') # pylint: disable=W1401 match = profile_re.search(temp) if match: @@ -187,6 +211,14 @@ def ext_pillar(minion_id, else: opts['profile'] = None + parse_yaml_re = re.compile('parse_yaml=False', re.IGNORECASE) # pylint: disable=W1401 + match = parse_yaml_re.search(temp) + if match: + opts['parse_yaml'] = False + temp = temp.replace(match.group(0), '') + else: + opts['parse_yaml'] = True + client = get_conn(__opts__, opts['profile']) role = __salt__['grains.get']('role', None) @@ -199,7 +231,10 @@ def ext_pillar(minion_id, } try: - pillar = fetch_tree(client, opts['root']) + pillar = fetch_tree(client, opts['root'], opts['parse_yaml']) + if opts['pillar_root']: + log.debug('Merging consul path %s/ into pillar at %s/', opts['root'], opts['pillar_root']) + pillar = nest_tree(pillar, opts['pillar_root']) except KeyError: log.error('No such key in consul profile %s: %s', opts['profile'], opts['root']) pillar = {} @@ -214,7 +249,7 @@ def consul_fetch(client, path): return client.kv.get(path, recurse=True) -def fetch_tree(client, path): +def fetch_tree(client, path, parse_yaml): ''' Grab data from consul, trim base path and remove any keys which are folders. Take the remaining data and send it to be formatted @@ -234,13 +269,13 @@ def fetch_tree(client, path): log.debug('key/path - %s: %s', path, key) log.debug('has_children? %r', has_children.search(key)) if has_children.search(key) is None: - ret = pillar_format(ret, key.split('/'), item['Value']) + ret = pillar_format(ret, key.split('/'), item['Value'], parse_yaml) log.debug('Fetching subkeys for key: %r', item) return ret -def pillar_format(ret, keys, value): +def pillar_format(ret, keys, value, parse_yaml): ''' Perform data formatting to be used as pillar data and merge it with the current pillar data @@ -250,9 +285,12 @@ def pillar_format(ret, keys, value): return ret # If value is not None then it's a string - # Use YAML to parse the data + # If parse_yaml is true, Use YAML to parse the data # YAML strips whitespaces unless they're surrounded by quotes - pillar_value = salt.utils.yaml.safe_load(value) + if parse_yaml: + pillar_value = salt.utils.yaml.safe_load(value) + else: + pillar_value = value keyvalue = keys.pop() pil = {keyvalue: pillar_value} @@ -263,6 +301,24 @@ def pillar_format(ret, keys, value): return dict_merge(ret, pil) +def nest_tree(pillar, pillar_root): + ''' + Returns a pillar nested in a subkey + ''' + ret = {} + branch = ret + keys = pillar_root.split('/') + + for i, k in enumerate(keys): + if i == len(keys) - 1: + branch[k] = pillar + else: + branch[k] = {} + branch = branch[k] + + return ret + + def get_conn(opts, profile): ''' From 7322b8c0a0c521c103d34d9512535f51e2020b33 Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Fri, 29 Jun 2018 16:28:23 -0700 Subject: [PATCH 598/769] rename parse_yaml to expand_keys --- src/saltext/consul/pillar/consul_pillar.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 4b6df47..5114a44 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -127,12 +127,12 @@ - consul: my_consul_config pillar_root=consul_data By default, keys containing YAML data will be deserialized before being merged into Pillar. -This behavior can be disabled by setting ``parse_yaml`` to ``false``. +This behavior can be disabled by setting ``expand_keys`` to ``false``. .. code-block:: yaml ext_pillar: - - consul: my_consul_config parse_yaml=false + - consul: my_consul_config expand_keys=false ''' from __future__ import absolute_import, print_function, unicode_literals @@ -211,13 +211,13 @@ def ext_pillar(minion_id, else: opts['profile'] = None - parse_yaml_re = re.compile('parse_yaml=False', re.IGNORECASE) # pylint: disable=W1401 - match = parse_yaml_re.search(temp) + expand_keys_re = re.compile('expand_keys=False', re.IGNORECASE) # pylint: disable=W1401 + match = expand_keys_re.search(temp) if match: - opts['parse_yaml'] = False + opts['expand_keys'] = False temp = temp.replace(match.group(0), '') else: - opts['parse_yaml'] = True + opts['expand_keys'] = True client = get_conn(__opts__, opts['profile']) @@ -231,7 +231,7 @@ def ext_pillar(minion_id, } try: - pillar = fetch_tree(client, opts['root'], opts['parse_yaml']) + pillar = fetch_tree(client, opts['root'], opts['expand_keys']) if opts['pillar_root']: log.debug('Merging consul path %s/ into pillar at %s/', opts['root'], opts['pillar_root']) pillar = nest_tree(pillar, opts['pillar_root']) @@ -249,7 +249,7 @@ def consul_fetch(client, path): return client.kv.get(path, recurse=True) -def fetch_tree(client, path, parse_yaml): +def fetch_tree(client, path, expand_keys): ''' Grab data from consul, trim base path and remove any keys which are folders. Take the remaining data and send it to be formatted @@ -269,13 +269,13 @@ def fetch_tree(client, path, parse_yaml): log.debug('key/path - %s: %s', path, key) log.debug('has_children? %r', has_children.search(key)) if has_children.search(key) is None: - ret = pillar_format(ret, key.split('/'), item['Value'], parse_yaml) + ret = pillar_format(ret, key.split('/'), item['Value'], expand_keys) log.debug('Fetching subkeys for key: %r', item) return ret -def pillar_format(ret, keys, value, parse_yaml): +def pillar_format(ret, keys, value, expand_keys): ''' Perform data formatting to be used as pillar data and merge it with the current pillar data @@ -285,9 +285,9 @@ def pillar_format(ret, keys, value, parse_yaml): return ret # If value is not None then it's a string - # If parse_yaml is true, Use YAML to parse the data + # If expand_keys is true, deserialize the YAML data # YAML strips whitespaces unless they're surrounded by quotes - if parse_yaml: + if expand_keys: pillar_value = salt.utils.yaml.safe_load(value) else: pillar_value = value From 15cda85709b72ca35b9da366fa09666985b3f393 Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Fri, 29 Jun 2018 16:30:42 -0700 Subject: [PATCH 599/769] reorder comment --- src/saltext/consul/pillar/consul_pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 5114a44..89754ed 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -285,8 +285,8 @@ def pillar_format(ret, keys, value, expand_keys): return ret # If value is not None then it's a string - # If expand_keys is true, deserialize the YAML data # YAML strips whitespaces unless they're surrounded by quotes + # If expand_keys is true, deserialize the YAML data if expand_keys: pillar_value = salt.utils.yaml.safe_load(value) else: From b430c38acd65f17284430295ddaa43bf78079ad5 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Jun 2018 15:15:32 -0700 Subject: [PATCH 600/769] When pillar items are compiled a new render is instantiated but the file_roots is the pillar_roots. This change forces the __opts__['file_roots'] to be set to what is set in actual_file_roots for all renderers once compile_pillar has finished. Adding a test when this situation is run via a orchestration runner. --- src/saltext/consul/pillar/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 63bc758..f0171ca 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -923,6 +923,11 @@ def compile_pillar(self, ext=True, pillar_dirs=None): decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: pillar.setdefault('_errors', []).extend(decrypt_errors) + + # Reset the file_roots for the renderers + for mod_name in sys.modules: + if mod_name.startswith('salt.loaded.int.render.'): + sys.modules[mod_name].__opts__['file_roots'] = self.actual_file_roots return pillar def decrypt_pillar(self, pillar): From c2e7fa07c020696b2ab295d8cc4bc8304cf6afe1 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Jun 2018 15:31:54 -0700 Subject: [PATCH 601/769] When pillar items are compiled a new render is instantiated but the file_roots is the pillar_roots. This change forces the __opts__['file_roots'] to be set to what is set in actual_file_roots for all renderers once compile_pillar has finished. Adding a test when this situation is run via a orchestration runner. --- src/saltext/consul/pillar/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fc3ce0a..210f2bb 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1037,6 +1037,11 @@ def compile_pillar(self, ext=True): decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: pillar.setdefault('_errors', []).extend(decrypt_errors) + + # Reset the file_roots for the renderers + for mod_name in sys.modules: + if mod_name.startswith('salt.loaded.int.render.'): + sys.modules[mod_name].__opts__['file_roots'] = self.actual_file_roots return pillar def decrypt_pillar(self, pillar): From 5707d0f20eaa29f151767bbc2290ea238b3ea06f Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Tue, 10 Jul 2018 16:42:09 -0700 Subject: [PATCH 602/769] * moved pillar nesting into ext_pillar() * added test case for pillar key nesting * added test case for disabling yaml deserialization --- src/saltext/consul/pillar/consul_pillar.py | 34 +++++++++------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 89754ed..430d356 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -231,10 +231,22 @@ def ext_pillar(minion_id, } try: - pillar = fetch_tree(client, opts['root'], opts['expand_keys']) + pillar_tree = fetch_tree(client, opts['root'], opts['expand_keys']) if opts['pillar_root']: log.debug('Merging consul path %s/ into pillar at %s/', opts['root'], opts['pillar_root']) - pillar = nest_tree(pillar, opts['pillar_root']) + + pillar = {} + branch = pillar + keys = opts['pillar_root'].rstrip('/').split('/') + + for i, k in enumerate(keys): + if i == len(keys) - 1: + branch[k] = pillar_tree + else: + branch[k] = {} + branch = branch[k] + else: + pillar = pillar_tree except KeyError: log.error('No such key in consul profile %s: %s', opts['profile'], opts['root']) pillar = {} @@ -301,24 +313,6 @@ def pillar_format(ret, keys, value, expand_keys): return dict_merge(ret, pil) -def nest_tree(pillar, pillar_root): - ''' - Returns a pillar nested in a subkey - ''' - ret = {} - branch = ret - keys = pillar_root.split('/') - - for i, k in enumerate(keys): - if i == len(keys) - 1: - branch[k] = pillar - else: - branch[k] = {} - branch = branch[k] - - return ret - - def get_conn(opts, profile): ''' From ab9e5078626fbbe7ef86aca523eeb4fbf5d028d7 Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Fri, 20 Jul 2018 14:03:05 -0700 Subject: [PATCH 603/769] removed trailing whitespace for pylint --- src/saltext/consul/pillar/consul_pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 430d356..5038fba 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -126,7 +126,7 @@ ext_pillar: - consul: my_consul_config pillar_root=consul_data -By default, keys containing YAML data will be deserialized before being merged into Pillar. +By default, keys containing YAML data will be deserialized before being merged into Pillar. This behavior can be disabled by setting ``expand_keys`` to ``false``. .. code-block:: yaml From 6070dd39947f70a39f76bcf5cb3b7acdefde8c18 Mon Sep 17 00:00:00 2001 From: Doug Clow Date: Fri, 20 Jul 2018 14:10:12 -0700 Subject: [PATCH 604/769] assign index to '_' in fetch_tree return for pylint --- src/saltext/consul/pillar/consul_pillar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 5038fba..e725fd4 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -267,7 +267,7 @@ def fetch_tree(client, path, expand_keys): are folders. Take the remaining data and send it to be formatted in such a way as to be used as pillar data. ''' - index, items = consul_fetch(client, path) + _, items = consul_fetch(client, path) ret = {} has_children = re.compile(r'/$') From 77fedb13efd7ce0981508dd4f7c5375630017d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Jouannet?= Date: Tue, 21 Aug 2018 17:49:47 +0200 Subject: [PATCH 605/769] fix HTTP method for acl_info --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 48a9997..57d3fb8 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -2164,7 +2164,7 @@ def acl_info(consul_url=None, **kwargs): function = 'acl/info/{0}'.format(kwargs['id']) ret = _query(consul_url=consul_url, data=data, - method='PUT', + method='GET', function=function) return ret From f9182d1541439ae12a40e690c1afd116f96d9e37 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 10 Sep 2018 13:19:03 -0600 Subject: [PATCH 606/769] Kill bare excepts with fire These prevent the signal handlers from shutting down a process via a raising a SystemExit. They are ticking time bombs of race conditions that leave processes running when daemons are shut down or restarted. Friends don't let friends use bare excepts. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index f0171ca..5d22aa2 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -112,7 +112,7 @@ def compile_pillar(self): load, dictkey='pillar', ) - except: + except Exception: log.exception('Exception getting pillar:') raise SaltClientError('Exception getting pillar.') From 2848edc008ddb24c2545ed4853d3b75af1075a3e Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Tue, 14 Aug 2018 16:01:39 -0600 Subject: [PATCH 607/769] Remove Matcher class and use the bare loader. --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0a7bf9c..5964e0f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -366,7 +366,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, else: self.functions = functions - self.matcher = salt.minion.Matcher(self.opts, self.functions) + self.matchers = salt.loader.matchers(self.opts) self.rend = salt.loader.render(self.opts, self.functions) ext_pillar_opts = copy.deepcopy(self.opts) # Fix self.opts['file_roots'] so that ext_pillars know the real @@ -649,7 +649,7 @@ def top_matches(self, top): if saltenv != self.opts['pillarenv']: continue for match, data in six.iteritems(body): - if self.matcher.confirm_top( + if self.matchers['confirm_top.confirm_top']( match, data, self.opts.get('nodegroups', {}), From d4463c4193abc6d0f1fa4e09ed923b9e17a5ab1f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 10 Oct 2018 13:10:50 -0500 Subject: [PATCH 608/769] Make Pillar no longer munge file_roots Pillar and the local fileclient used to use the same class, but this changed a few years ago with the introduction of FSClient (which enabled non-roots fileserver backends in masterless Salt). Since Pillar is the only thing that uses this client, there is no longer a reason to make Pillar modify file_roots. --- src/saltext/consul/pillar/__init__.py | 28 ++++++++------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4194ff8..b7dad9e 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -345,8 +345,6 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if pillarenv is None: if opts.get('pillarenv_from_saltenv', False): opts['pillarenv'] = saltenv - # Store the file_roots path so we can restore later. Issue 5449 - self.actual_file_roots = opts['file_roots'] # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) self.saltenv = saltenv @@ -369,9 +367,6 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.matcher = salt.minion.Matcher(self.opts, self.functions) self.rend = salt.loader.render(self.opts, self.functions) ext_pillar_opts = copy.deepcopy(self.opts) - # Fix self.opts['file_roots'] so that ext_pillars know the real - # location of file_roots. Issue 5951 - ext_pillar_opts['file_roots'] = self.actual_file_roots # Keep the incoming opts ID intact, ie, the master id if 'id' in opts: ext_pillar_opts['id'] = opts['id'] @@ -438,7 +433,6 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): The options need to be altered to conform to the file client ''' opts = copy.deepcopy(opts_in) - opts['file_roots'] = opts['pillar_roots'] opts['file_client'] = 'local' if not grains: opts['grains'] = {} @@ -463,15 +457,15 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): opts['ext_pillar'].append(self.ext) else: opts['ext_pillar'] = [self.ext] - if '__env__' in opts['file_roots']: + if '__env__' in opts['pillar_roots']: env = opts.get('pillarenv') or opts.get('saltenv') or 'base' - if env not in opts['file_roots']: + if env not in opts['pillar_roots']: log.debug("pillar environment '%s' maps to __env__ pillar_roots directory", env) - opts['file_roots'][env] = opts['file_roots'].pop('__env__') + opts['pillar_roots'][env] = opts['pillar_roots'].pop('__env__') else: log.debug("pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", env) - opts['file_roots'].pop('__env__') + opts['pillar_roots'].pop('__env__') return opts def _get_envs(self): @@ -479,8 +473,8 @@ def _get_envs(self): Pull the file server environments out of the master options ''' envs = set(['base']) - if 'file_roots' in self.opts: - envs.update(list(self.opts['file_roots'])) + if 'pillar_roots' in self.opts: + envs.update(list(self.opts['pillar_roots'])) return envs def get_tops(self): @@ -496,11 +490,11 @@ def get_tops(self): if self.opts['pillarenv']: # If the specified pillarenv is not present in the available # pillar environments, do not cache the pillar top file. - if self.opts['pillarenv'] not in self.opts['file_roots']: + if self.opts['pillarenv'] not in self.opts['pillar_roots']: log.debug( 'pillarenv \'%s\' not found in the configured pillar ' 'environments (%s)', - self.opts['pillarenv'], ', '.join(self.opts['file_roots']) + self.opts['pillarenv'], ', '.join(self.opts['pillar_roots']) ) else: top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) @@ -1010,8 +1004,6 @@ def compile_pillar(self, ext=True): mopts = dict(self.opts) if 'grains' in mopts: mopts.pop('grains') - # Restore the actual file_roots path. Issue 5449 - mopts['file_roots'] = self.actual_file_roots mopts['saltversion'] = __version__ pillar['master'] = mopts if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False): @@ -1038,10 +1030,6 @@ def compile_pillar(self, ext=True): if decrypt_errors: pillar.setdefault('_errors', []).extend(decrypt_errors) - # Reset the file_roots for the renderers - for mod_name in sys.modules: - if mod_name.startswith('salt.loaded.int.render.'): - sys.modules[mod_name].__opts__['file_roots'] = self.actual_file_roots return pillar def decrypt_pillar(self, pillar): From 31215446681a50cba782b3e23e3ccbd0faac5eec Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 15 Oct 2018 13:29:54 -0400 Subject: [PATCH 609/769] [2018.3] Fix some pylint issues that have popped up recently --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 4c7bcf8..5e73e3e 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -214,7 +214,7 @@ def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw= if ret['res']: if decode: for item in ret['data']: - if item['Value'] != None: + if item['Value'] is not None: item['Value'] = base64.b64decode(item['Value']) else: item['Value'] = "" From 1bed173ac80346a6dffea8f7f0df6666cf528a3d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 28 Dec 2018 11:59:43 +0000 Subject: [PATCH 610/769] Stop using the deprecated `salt.transport.Channel.factory` --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 929cc17..427fc41 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -20,7 +20,7 @@ import salt.fileclient import salt.minion import salt.crypt -import salt.transport +import salt.transport.client import salt.utils.args import salt.utils.cache import salt.utils.crypt @@ -202,7 +202,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.ext = ext self.grains = grains self.minion_id = minion_id - self.channel = salt.transport.Channel.factory(opts) + self.channel = salt.transport.client.ReqChannel.factory(opts) if pillarenv is not None: self.opts['pillarenv'] = pillarenv self.pillar_override = pillar_override or {} From ee5e523078a8fc34a36ea593ad40d54d4221e203 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 28 Dec 2018 11:21:47 +0000 Subject: [PATCH 611/769] Make sure we close the pillar channel --- src/saltext/consul/pillar/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 427fc41..676b362 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -157,6 +157,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.get_ext_pillar_extra_minion_data(opts), recursive_update=True, merge_lists=True) + self._closing = False @tornado.gen.coroutine def compile_pillar(self): @@ -190,6 +191,16 @@ def compile_pillar(self): raise SaltClientError(msg) raise tornado.gen.Return(ret_pillar) + def destroy(self): + if self._closing: + return + + self._closing = True + self.channel.close() + + def __del__(self): + self.destroy() + class RemotePillar(RemotePillarMixin): ''' @@ -217,6 +228,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.get_ext_pillar_extra_minion_data(opts), recursive_update=True, merge_lists=True) + self._closing = False def compile_pillar(self): ''' @@ -244,6 +256,16 @@ def compile_pillar(self): return {} return ret_pillar + def destroy(self): + if self._closing: + return + + self._closing = True + self.channel.close() + + def __del__(self): + self.destroy() + class PillarCache(object): ''' From 331862003ae1880b62e8672a686e1a00f7d9f40a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 17 Jan 2019 17:14:43 +0000 Subject: [PATCH 612/769] Make `AsyncPillar` API compatible with `AsyncRemotePillar` Fixes #51186 --- src/saltext/consul/pillar/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 676b362..94dd969 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -406,6 +406,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, if not isinstance(self.extra_minion_data, dict): self.extra_minion_data = {} log.error('Extra minion data must be a dictionary') + self._closing = False def __valid_on_demand_ext_pillar(self, opts): ''' @@ -1109,6 +1110,17 @@ def decrypt_pillar(self, pillar): log.error(msg, exc_info=True) return errors + def destroy(self): + ''' + This method exist in order to be API compatible with RemotePillar + ''' + if self._closing: + return + self._closing = True + + def __del__(self): + self.destroy() + # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in # ext_pillar etc. From d7ac91da6aa1bc1b14d50665078e068b7a10a47b Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 28 Jan 2019 10:53:10 -0600 Subject: [PATCH 613/769] Add Python2 depreciation warning --- src/saltext/consul/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 22f8145..f34eff3 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -29,7 +29,6 @@ UserWarning ) - def __define_global_system_encoding_variable__(): import sys # This is the most trustworthy source of the system encoding, though, if @@ -95,3 +94,17 @@ def __define_global_system_encoding_variable__(): # This is now garbage collectable del __define_global_system_encoding_variable__ + + +# REMOVEME after Python 2.7 support is dropped +import six + +if six.PY2: + from salt.utils.versions import warn_until + # Message borrowed from pip's deprecation warning + warn_until('Sodium', + 'Python 2.7 will reach the end of its life on January 1st, 2020.' + ' Please upgrade your Python as Python 2.7 won\'t be maintained' + ' after that date. Salt will drop support for Python 2.7 in the' + ' Sodium release.') +# END REMOVEME From 9c68dd4c0225f43af0450ed859d12af34af03f1a Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 28 Jan 2019 14:39:27 -0600 Subject: [PATCH 614/769] Fix lint issues --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f34eff3..56f1edc 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -29,6 +29,7 @@ UserWarning ) + def __define_global_system_encoding_variable__(): import sys # This is the most trustworthy source of the system encoding, though, if @@ -97,7 +98,7 @@ def __define_global_system_encoding_variable__(): # REMOVEME after Python 2.7 support is dropped -import six +import salt.ext.six as six if six.PY2: from salt.utils.versions import warn_until From 5b95a10afca471af2c0c747b4e6ad95771d519fb Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 28 Jan 2019 15:29:44 -0600 Subject: [PATCH 615/769] Quieter warnings for now. We want to throw them in now for salt master/minion startup, but by the sodium release we'll want to put them back in `salt/__init__.py`, because if they haven't upgraded to Python3 by that point then they really really need to get on the ball! --- src/saltext/consul/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 56f1edc..22f8145 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -95,17 +95,3 @@ def __define_global_system_encoding_variable__(): # This is now garbage collectable del __define_global_system_encoding_variable__ - - -# REMOVEME after Python 2.7 support is dropped -import salt.ext.six as six - -if six.PY2: - from salt.utils.versions import warn_until - # Message borrowed from pip's deprecation warning - warn_until('Sodium', - 'Python 2.7 will reach the end of its life on January 1st, 2020.' - ' Please upgrade your Python as Python 2.7 won\'t be maintained' - ' after that date. Salt will drop support for Python 2.7 in the' - ' Sodium release.') -# END REMOVEME From 7f609ce5c80c8f5f698e4fcf05262a5a01305eeb Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 18 Mar 2019 18:13:54 -0700 Subject: [PATCH 616/769] With the change/addition of the matcher subsystem in 2019.2, the match.search_by when used in pillar broke when targetting the minion that was also the salt master. This was caused by the id in __opts__ being used in all cases. This change updates the glob_match function to use the preserved minion_id of the master if it is available so that targeting works as expected. --- src/saltext/consul/pillar/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 94dd969..f20f4ec 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -386,6 +386,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, else: self.functions = functions + self.opts['minion_id'] = minion_id self.matchers = salt.loader.matchers(self.opts) self.rend = salt.loader.render(self.opts, self.functions) ext_pillar_opts = copy.deepcopy(self.opts) From ce702a6526cbc38e0a89398d7017ba748e412438 Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Fri, 5 Apr 2019 15:37:54 +0200 Subject: [PATCH 617/769] Also remove unused salt.crypt import in pillar/__init__.py --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 94dd969..f2a1a86 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -19,7 +19,6 @@ import salt.loader import salt.fileclient import salt.minion -import salt.crypt import salt.transport.client import salt.utils.args import salt.utils.cache From 776edde4498c86cd4215f426c34ee3bc68f4d93b Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Wed, 6 Mar 2019 14:00:03 -0600 Subject: [PATCH 618/769] Swap '/' for '.' when matching --- src/saltext/consul/pillar/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 94dd969..6fe2b5f 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -774,7 +774,10 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key = None try: - matched_pstates += fnmatch.filter(self.avail[saltenv], sub_sls) + matched_pstates.extend(fnmatch.filter( + self.avail[saltenv], + sub_sls.replace('/', '.'), + )) except KeyError: errors.extend( ['No matching pillar environment for environment ' From 02b5911b24b7e57e3acf367759933bf21f6e4bbb Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Fri, 8 Mar 2019 12:56:25 -0600 Subject: [PATCH 619/769] Ensure _closing exists This was raising an ignored exception because _closing was set later in __init__. Not sure if it must be the last line in __init__. --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6fe2b5f..4899ac3 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -257,7 +257,7 @@ def compile_pillar(self): return ret_pillar def destroy(self): - if self._closing: + if hasattr(self, '_closing') and self._closing: return self._closing = True From 98f1f77ac911840b699da40e07c721a03a9b7302 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 11 Mar 2019 15:11:55 -0500 Subject: [PATCH 620/769] Allow leading dots and / as pillar separators --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4899ac3..38225ca 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -776,7 +776,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): try: matched_pstates.extend(fnmatch.filter( self.avail[saltenv], - sub_sls.replace('/', '.'), + sub_sls.lstrip('.').replace('/', '.'), )) except KeyError: errors.extend( From 675e4927e0ec47df6cab588624551831f8faf3d5 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 10 Apr 2019 20:51:19 +0000 Subject: [PATCH 621/769] Revert "Fix #22063: pillar wildcard support include" This reverts commit f50c304ad9b14ca9e3c1bda889b1c8568ac94e22. --- src/saltext/consul/pillar/__init__.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 94dd969..3f03693 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -763,8 +763,6 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): else: # render included state(s) include_states = [] - - matched_pstates = [] for sub_sls in state.pop('include'): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) @@ -772,16 +770,6 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key = v.get('key', None) else: key = None - - try: - matched_pstates += fnmatch.filter(self.avail[saltenv], sub_sls) - except KeyError: - errors.extend( - ['No matching pillar environment for environment ' - '\'{0}\' found'.format(saltenv)] - ) - - for sub_sls in set(matched_pstates): if sub_sls not in mods: nstate, mods, err = self.render_pstate( sub_sls, From 31868b93ab29cca06f5f0a0402383cc00730ad4c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 10 Apr 2019 20:56:07 +0000 Subject: [PATCH 622/769] Fix pillar include wart --- src/saltext/consul/pillar/__init__.py | 60 +++++++++++++++------------ 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3f03693..61e7ac4 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -770,32 +770,40 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): key = v.get('key', None) else: key = None - if sub_sls not in mods: - nstate, mods, err = self.render_pstate( - sub_sls, - saltenv, - mods, - defaults - ) - if nstate: - if key: - # If key is x:y, convert it to {x: {y: nstate}} - for key_fragment in reversed(key.split(":")): - nstate = { - key_fragment: nstate - } - if not self.opts.get('pillar_includes_override_sls', False): - include_states.append(nstate) - else: - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) - if err: - errors += err - + try: + matched_pstates = fnmatch.filter(self.avail[saltenv], sub_sls) + except KeyError: + errors.extend( + ['No matching pillar environment for environment ' + '\'{0}\' found'.format(saltenv)] + ) + matched_pstates = [sub_sls] + for m_sub_sls in matched_pstates: + if m_sub_sls not in mods: + nstate, mods, err = self.render_pstate( + m_sub_sls, + saltenv, + mods, + defaults + ) + if nstate: + if key: + # If key is x:y, convert it to {x: {y: nstate}} + for key_fragment in reversed(key.split(":")): + nstate = { + key_fragment: nstate + } + if not self.opts.get('pillar_includes_override_sls', False): + include_states.append(nstate) + else: + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get('renderer', 'yaml'), + self.opts.get('pillar_merge_lists', False)) + if err: + errors += err if not self.opts.get('pillar_includes_override_sls', False): # merge included state(s) with the current state # merged last to ensure that its values are From 65372b24e6144cc16826d21168985a5044137bb1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 12 Apr 2019 03:58:00 +0000 Subject: [PATCH 623/769] Fix merge wart --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index e95e911..4592ac0 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -774,7 +774,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): matched_pstates = fnmatch.filter( self.avail[saltenv], sub_sls.lstrip('.').replace('/', '.'), - )) + ) except KeyError: errors.extend( ['No matching pillar environment for environment ' From 9277035748c086611bdd485b99790321611b8ee9 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 6 May 2019 17:57:11 -0400 Subject: [PATCH 624/769] Reload matcher loader when ext_pillar_first set --- src/saltext/consul/pillar/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4592ac0..fc9a2a8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -655,7 +655,7 @@ def get_top(self): errors.append('Error encountered while rendering pillar top file.') return merged_tops, errors - def top_matches(self, top): + def top_matches(self, top, reload=False): ''' Search through the top high data for matches and return the states that this minion needs to execute. @@ -664,6 +664,8 @@ def top_matches(self, top): {'saltenv': ['state1', 'state2', ...]} ''' matches = {} + if reload: + self.matchers = salt.loader.matchers(self.opts) for saltenv, body in six.iteritems(top): if self.opts['pillarenv']: if saltenv != self.opts['pillarenv']: @@ -1001,7 +1003,7 @@ def compile_pillar(self, ext=True): if self.opts.get('ext_pillar_first', False): self.opts['pillar'], errors = self.ext_pillar(self.pillar_override) self.rend = salt.loader.render(self.opts, self.functions) - matches = self.top_matches(top) + matches = self.top_matches(top, reload=True) pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge( self.opts['pillar'], From 907a86cea6c8fc99fe4dd46b502266c5dc7b1cb8 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 6 May 2019 18:01:05 -0400 Subject: [PATCH 625/769] Add docs for new reload kwarg --- src/saltext/consul/pillar/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index fc9a2a8..13fed45 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -662,6 +662,9 @@ def top_matches(self, top, reload=False): Returns: {'saltenv': ['state1', 'state2', ...]} + + reload + Reload the matcher loader ''' matches = {} if reload: From e92192ab79570287210ff46ca22c65a4c193fe81 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 18 Sep 2019 17:39:25 -0700 Subject: [PATCH 626/769] Porting PR #49903 to 2019.2.1 --- src/saltext/consul/pillar/consul_pillar.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index e725fd4..6903ccd 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -187,10 +187,10 @@ def ext_pillar(minion_id, if minion_id not in minions: return {} - root_re = re.compile('root=(\S*)') # pylint: disable=W1401 + root_re = re.compile('(? Date: Fri, 6 Dec 2019 09:49:52 -0500 Subject: [PATCH 627/769] fix pylint for __del__ --- src/saltext/consul/pillar/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 13fed45..b7679fa 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -197,8 +197,10 @@ def destroy(self): self._closing = True self.channel.close() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class RemotePillar(RemotePillarMixin): @@ -262,8 +264,10 @@ def destroy(self): self._closing = True self.channel.close() + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class PillarCache(object): @@ -1122,8 +1126,10 @@ def destroy(self): return self._closing = True + # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in From 014cb103d37b10c565cde007a6a68ff99512a909 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 2 Jan 2020 13:42:37 +0000 Subject: [PATCH 628/769] Fix PyLint `broad-except` --- src/saltext/consul/cache/consul.py | 10 +++++----- src/saltext/consul/pillar/__init__.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 2f8e878..3bba9c5 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -100,7 +100,7 @@ def store(bank, key, data): try: c_data = __context__['serial'].dumps(data) api.kv.put(c_key, c_data) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( 'There was an error writing the key, {0}: {1}'.format( c_key, exc @@ -118,7 +118,7 @@ def fetch(bank, key): if value is None: return {} return __context__['serial'].loads(value['Value']) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( 'There was an error reading the key, {0}: {1}'.format( c_key, exc @@ -136,7 +136,7 @@ def flush(bank, key=None): c_key = '{0}/{1}'.format(bank, key) try: return api.kv.delete(c_key, recurse=key is None) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( 'There was an error removing the key, {0}: {1}'.format( c_key, exc @@ -150,7 +150,7 @@ def list_(bank): ''' try: _, keys = api.kv.get(bank + '/', keys=True, separator='/') - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( 'There was an error getting the key "{0}": {1}'.format( bank, exc @@ -178,7 +178,7 @@ def contains(bank, key): try: c_key = '{0}/{1}'.format(bank, key) _, value = api.kv.get(c_key) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( 'There was an error getting the key, {0}: {1}'.format( c_key, exc diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b7679fa..629ca94 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -178,7 +178,7 @@ def compile_pillar(self): load, dictkey='pillar', ) - except Exception: + except Exception: # pylint: disable=broad-except log.exception('Exception getting pillar:') raise SaltClientError('Exception getting pillar.') @@ -543,7 +543,7 @@ def get_tops(self): saltenv=saltenv, _pillar_rend=True, )) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except errors.append( ('Rendering Primary Top file failed, render error:\n{0}' .format(exc))) @@ -582,7 +582,7 @@ def get_tops(self): _pillar_rend=True, ) ) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except errors.append( ('Rendering Top file {0} failed, render error' ':\n{1}').format(sls, exc)) @@ -743,7 +743,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): sls, _pillar_rend=True, **defaults) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( sls, exc ) @@ -980,7 +980,7 @@ def ext_pillar(self, pillar, errors=None): ext = self._external_pillar_data(pillar, val, key) - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except errors.append( 'Failed to load ext_pillar {0}: {1}'.format( key, @@ -1110,7 +1110,7 @@ def decrypt_pillar(self, pillar): delimiter=self.opts['decrypt_pillar_delimiter']) if ptr is not None: ptr[child] = ret - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except msg = 'Failed to decrypt pillar key \'{0}\': {1}'.format( key, exc ) From 36f0eeed3b68d696a8d7c83fbe862ee195573918 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 23 Jan 2020 22:27:01 +0000 Subject: [PATCH 629/769] Use vendored tornado instead of system tornado --- src/saltext/consul/pillar/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 629ca94..6b36cda 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -10,7 +10,7 @@ import os import collections import logging -import tornado.gen +import salt.ext.tornado.gen import sys import traceback import inspect @@ -158,7 +158,7 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, merge_lists=True) self._closing = False - @tornado.gen.coroutine + @salt.ext.tornado.gen.coroutine def compile_pillar(self): ''' Return a future which will contain the pillar data from the master @@ -188,7 +188,7 @@ def compile_pillar(self): log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! raise SaltClientError(msg) - raise tornado.gen.Return(ret_pillar) + raise salt.ext.tornado.gen.Return(ret_pillar) def destroy(self): if self._closing: @@ -1135,7 +1135,7 @@ def __del__(self): # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in # ext_pillar etc. class AsyncPillar(Pillar): - @tornado.gen.coroutine + @salt.ext.tornado.gen.coroutine def compile_pillar(self, ext=True): ret = super(AsyncPillar, self).compile_pillar(ext=ext) - raise tornado.gen.Return(ret) + raise salt.ext.tornado.gen.Return(ret) From 4b21b1269515b7984551938a9abab60bfabcdcc3 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 28 Jan 2020 15:43:33 +0000 Subject: [PATCH 630/769] Always import vendored tornado --- src/saltext/consul/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 22f8145..57dd0ac 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -6,6 +6,20 @@ # Import Python libs from __future__ import absolute_import, print_function, unicode_literals import warnings +import sys +import importlib + +class TornadoImporter(object): + + def find_module(self, module_name, package_path): + if module_name.startswith('tornado'): + return self + return None + + def load_module(self, name): + return importlib.import_module('salt.ext.{}'.format(name)) + +sys.meta_path.append(TornadoImporter()) # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( From 85a4479435b3bb8d22f15dc931d8048b7a4b3044 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 28 Jan 2020 18:07:46 +0000 Subject: [PATCH 631/769] Make importer py3 compatible --- src/saltext/consul/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 57dd0ac..f0673aa 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -8,6 +8,9 @@ import warnings import sys import importlib +import imp +import salt.ext.six + class TornadoImporter(object): @@ -17,9 +20,14 @@ def find_module(self, module_name, package_path): return None def load_module(self, name): - return importlib.import_module('salt.ext.{}'.format(name)) + mod = importlib.import_module('salt.ext.{}'.format(name)) + sys.modules[name] = mod + return mod + + +# Try our importer first +sys.meta_path = [TornadoImporter()] + sys.meta_path -sys.meta_path.append(TornadoImporter()) # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( From caa6eb26cb87ed816bdd0de5b81bedb3a16536fc Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 28 Jan 2020 18:20:01 +0000 Subject: [PATCH 632/769] Clean up un-used imports --- src/saltext/consul/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f0673aa..3e99d24 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -8,8 +8,6 @@ import warnings import sys import importlib -import imp -import salt.ext.six class TornadoImporter(object): From bc0f360bb547c88a72d5874ef8874a99caa30da5 Mon Sep 17 00:00:00 2001 From: Bryce Larson Date: Tue, 28 Jan 2020 13:15:26 -0700 Subject: [PATCH 633/769] fix syntax errors on code-block sections in docs --- src/saltext/consul/cache/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 3bba9c5..70c2f0c 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -15,7 +15,7 @@ To enable this cache plugin, the master will need the python client for Consul installed. This can be easily installed with pip: -.. code-block: bash +.. code-block:: bash pip install python-consul From 48fd88da823ea9406c155f56ac250d9e2be87a81 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 8 Feb 2020 02:08:04 +0000 Subject: [PATCH 634/769] Fix type error in TornadoImporter --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3e99d24..117523b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,7 +12,7 @@ class TornadoImporter(object): - def find_module(self, module_name, package_path): + def find_module(self, module_name, package_path=None): if module_name.startswith('tornado'): return self return None From 57f92b40e01ec6d98245c01d839ad9dcfa1a92f0 Mon Sep 17 00:00:00 2001 From: Blacken Salt Date: Thu, 2 Apr 2020 20:10:20 -0500 Subject: [PATCH 635/769] Blacken salt --- src/saltext/consul/__init__.py | 48 +- src/saltext/consul/cache/__init__.py | 91 +- src/saltext/consul/cache/consul.py | 100 +- src/saltext/consul/modules/__init__.py | 4 +- src/saltext/consul/modules/consul.py | 1866 ++++++++++---------- src/saltext/consul/pillar/__init__.py | 1012 ++++++----- src/saltext/consul/pillar/consul_pillar.py | 176 +- src/saltext/consul/sdb/__init__.py | 4 +- src/saltext/consul/sdb/consul.py | 21 +- src/saltext/consul/states/__init__.py | 4 +- 10 files changed, 1724 insertions(+), 1602 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 117523b..c620fb5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,24 +1,24 @@ # -*- coding: utf-8 -*- -''' +""" Salt package -''' +""" # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import warnings -import sys + import importlib +import sys +import warnings class TornadoImporter(object): - def find_module(self, module_name, package_path=None): - if module_name.startswith('tornado'): + if module_name.startswith("tornado"): return self return None def load_module(self, name): - mod = importlib.import_module('salt.ext.{}'.format(name)) + mod = importlib.import_module("salt.ext.{}".format(name)) sys.modules[name] = mod return mod @@ -29,35 +29,36 @@ def load_module(self, name): # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( - 'once', # Show once - '', # No deprecation message match + "once", # Show once + "", # No deprecation message match DeprecationWarning, # This filter is for DeprecationWarnings - r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' + r"^(salt|salt\.(.*))$", # Match module(s) 'salt' and 'salt.' ) # While we are supporting Python2.6, hide nested with-statements warnings warnings.filterwarnings( - 'ignore', - 'With-statements now directly support multiple context managers', - DeprecationWarning + "ignore", + "With-statements now directly support multiple context managers", + DeprecationWarning, ) # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( - 'ignore', - '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', - UserWarning + "ignore", + "^Module backports was already imported from (.*), but (.*) is being added to sys.path$", + UserWarning, ) def __define_global_system_encoding_variable__(): import sys + # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None encoding = None - if not sys.platform.startswith('win') and sys.stdin is not None: + if not sys.platform.startswith("win") and sys.stdin is not None: # On linux we can rely on sys.stdin for the encoding since it # most commonly matches the filesystem encoding. This however # does not apply to windows @@ -68,6 +69,7 @@ def __define_global_system_encoding_variable__(): # encoding. MS Windows has problems with this and reports the wrong # encoding import locale + try: encoding = locale.getdefaultlocale()[-1] except ValueError: @@ -83,16 +85,16 @@ def __define_global_system_encoding_variable__(): # the way back to ascii encoding = sys.getdefaultencoding() if not encoding: - if sys.platform.startswith('darwin'): + if sys.platform.startswith("darwin"): # Mac OS X uses UTF-8 - encoding = 'utf-8' - elif sys.platform.startswith('win'): + encoding = "utf-8" + elif sys.platform.startswith("win"): # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” # to refer to whatever the currently configured encoding is. - encoding = 'mbcs' + encoding = "mbcs" else: # On linux default to ascii as a last resort - encoding = 'ascii' + encoding = "ascii" # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: @@ -103,7 +105,7 @@ def __define_global_system_encoding_variable__(): import builtins # pylint: disable=import-error # Define the detected encoding as a built-in variable for ease of use - setattr(builtins, '__salt_system_encoding__', encoding) + setattr(builtins, "__salt_system_encoding__", encoding) # This is now garbage collectable del sys diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 68ffcdc..b7c6ca9 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -1,33 +1,34 @@ # -*- coding: utf-8 -*- -''' +""" Loader mechanism for caching data, with data expiration, etc. .. versionadded:: 2016.11.0 -''' +""" # Import Python libs from __future__ import absolute_import, print_function, unicode_literals + import logging import time # Import Salt libs import salt.config +import salt.loader +import salt.syspaths from salt.ext import six from salt.payload import Serial from salt.utils.odict import OrderedDict -import salt.loader -import salt.syspaths log = logging.getLogger(__name__) def factory(opts, **kwargs): - ''' + """ Creates and returns the cache class. If memory caching is enabled by opts MemCache class will be instantiated. If not Cache class will be returned. - ''' - if opts.get('memcache_expire_seconds', 0): + """ + if opts.get("memcache_expire_seconds", 0): cls = MemCache else: cls = Cache @@ -35,7 +36,7 @@ def factory(opts, **kwargs): class Cache(object): - ''' + """ Base caching object providing access to the modular cache subsystem. Related configuration options: @@ -66,22 +67,23 @@ class Cache(object): Key name is a string identifier of a data container (like a file inside a directory) which will hold the data. - ''' + """ + def __init__(self, opts, cachedir=None, **kwargs): self.opts = opts if cachedir is None: - self.cachedir = opts.get('cachedir', salt.syspaths.CACHE_DIR) + self.cachedir = opts.get("cachedir", salt.syspaths.CACHE_DIR) else: self.cachedir = cachedir - self.driver = opts.get('cache', salt.config.DEFAULT_MASTER_OPTS['cache']) + self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"]) self.serial = Serial(opts) self._modules = None self._kwargs = kwargs - self._kwargs['cachedir'] = self.cachedir + self._kwargs["cachedir"] = self.cachedir def __lazy_init(self): self._modules = salt.loader.cache(self.opts, self.serial) - fun = '{0}.init_kwargs'.format(self.driver) + fun = "{0}.init_kwargs".format(self.driver) if fun in self.modules: self._kwargs = self.modules[fun](self._kwargs) else: @@ -94,7 +96,7 @@ def modules(self): return self._modules def cache(self, bank, key, fun, loop_fun=None, **kwargs): - ''' + """ Check cache for the data. If it is there, check to see if it needs to be refreshed. @@ -106,8 +108,8 @@ def cache(self, bank, key, fun, loop_fun=None, **kwargs): the second function is passed in as ``loop_fun``. Each item in the return list from the first function will be the only argument for the second function. - ''' - expire_seconds = kwargs.get('expire', 86400) # 1 day + """ + expire_seconds = kwargs.get("expire", 86400) # 1 day updated = self.updated(bank, key) update_cache = False @@ -132,7 +134,7 @@ def cache(self, bank, key, fun, loop_fun=None, **kwargs): return data def store(self, bank, key, data): - ''' + """ Store data using the specified module :param bank: @@ -151,12 +153,12 @@ def store(self, bank, key, data): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.store'.format(self.driver) + """ + fun = "{0}.store".format(self.driver) return self.modules[fun](bank, key, data, **self._kwargs) def fetch(self, bank, key): - ''' + """ Fetch data using the specified module :param bank: @@ -175,12 +177,12 @@ def fetch(self, bank, key): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.fetch'.format(self.driver) + """ + fun = "{0}.fetch".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) def updated(self, bank, key): - ''' + """ Get the last updated epoch for the specified key :param bank: @@ -199,12 +201,12 @@ def updated(self, bank, key): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.updated'.format(self.driver) + """ + fun = "{0}.updated".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) def flush(self, bank, key=None): - ''' + """ Remove the key from the cache bank with all the key content. If no key is specified remove the entire bank with all keys and sub-banks inside. @@ -220,12 +222,12 @@ def flush(self, bank, key=None): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.flush'.format(self.driver) + """ + fun = "{0}.flush".format(self.driver) return self.modules[fun](bank, key=key, **self._kwargs) def list(self, bank): - ''' + """ Lists entries stored in the specified bank. :param bank: @@ -239,12 +241,12 @@ def list(self, bank): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.list'.format(self.driver) + """ + fun = "{0}.list".format(self.driver) return self.modules[fun](bank, **self._kwargs) def contains(self, bank, key=None): - ''' + """ Checks if the specified bank contains the specified key. :param bank: @@ -264,25 +266,26 @@ def contains(self, bank, key=None): :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). - ''' - fun = '{0}.contains'.format(self.driver) + """ + fun = "{0}.contains".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) class MemCache(Cache): - ''' + """ Short-lived in-memory cache store keeping values on time and/or size (count) basis. - ''' + """ + # {: odict({: [atime, data], ...}), ...} data = {} def __init__(self, opts, **kwargs): super(MemCache, self).__init__(opts, **kwargs) - self.expire = opts.get('memcache_expire_seconds', 10) - self.max = opts.get('memcache_max_items', 1024) - self.cleanup = opts.get('memcache_full_cleanup', False) - self.debug = opts.get('memcache_debug', False) + self.expire = opts.get("memcache_expire_seconds", 10) + self.max = opts.get("memcache_max_items", 1024) + self.cleanup = opts.get("memcache_full_cleanup", False) + self.debug = opts.get("memcache_debug", False) if self.debug: self.call = 0 self.hit = 0 @@ -299,7 +302,7 @@ def __cleanup(cls, expire): break def _get_storage_id(self): - fun = '{0}.storage_id'.format(self.driver) + fun = "{0}.storage_id".format(self.driver) if fun in self.modules: return self.modules[fun](self.kwargs) else: @@ -324,8 +327,10 @@ def fetch(self, bank, key): if self.debug: self.hit += 1 log.debug( - 'MemCache stats (call/hit/rate): %s/%s/%s', - self.call, self.hit, float(self.hit) / self.call + "MemCache stats (call/hit/rate): %s/%s/%s", + self.call, + self.hit, + float(self.hit) / self.call, ) # update atime and return record[0] = now diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 70c2f0c..8e61102 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Minion data cache plugin for Consul key/value data store. .. versionadded:: 2016.11.2 @@ -45,116 +45,118 @@ .. _`Consul documentation`: https://www.consul.io/docs/index.html .. _`python-consul documentation`: https://python-consul.readthedocs.io/en/latest/#consul -''' +""" from __future__ import absolute_import, print_function, unicode_literals + import logging + +from salt.exceptions import SaltCacheError + try: import consul + HAS_CONSUL = True except ImportError: HAS_CONSUL = False -from salt.exceptions import SaltCacheError log = logging.getLogger(__name__) api = None # Define the module's virtual name -__virtualname__ = 'consul' +__virtualname__ = "consul" -__func_alias__ = {'list_': 'list'} +__func_alias__ = {"list_": "list"} def __virtual__(): - ''' + """ Confirm this python-consul package is installed - ''' + """ if not HAS_CONSUL: - return (False, "Please install python-consul package to use consul data cache driver") + return ( + False, + "Please install python-consul package to use consul data cache driver", + ) consul_kwargs = { - 'host': __opts__.get('consul.host', '127.0.0.1'), - 'port': __opts__.get('consul.port', 8500), - 'token': __opts__.get('consul.token', None), - 'scheme': __opts__.get('consul.scheme', 'http'), - 'consistency': __opts__.get('consul.consistency', 'default'), - 'dc': __opts__.get('consul.dc', None), - 'verify': __opts__.get('consul.verify', True), - } + "host": __opts__.get("consul.host", "127.0.0.1"), + "port": __opts__.get("consul.port", 8500), + "token": __opts__.get("consul.token", None), + "scheme": __opts__.get("consul.scheme", "http"), + "consistency": __opts__.get("consul.consistency", "default"), + "dc": __opts__.get("consul.dc", None), + "verify": __opts__.get("consul.verify", True), + } try: global api api = consul.Consul(**consul_kwargs) except AttributeError: - return (False, "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed") + return ( + False, + "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed", + ) return __virtualname__ def store(bank, key, data): - ''' + """ Store a key value. - ''' - c_key = '{0}/{1}'.format(bank, key) + """ + c_key = "{0}/{1}".format(bank, key) try: - c_data = __context__['serial'].dumps(data) + c_data = __context__["serial"].dumps(data) api.kv.put(c_key, c_data) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error writing the key, {0}: {1}'.format( - c_key, exc - ) + "There was an error writing the key, {0}: {1}".format(c_key, exc) ) def fetch(bank, key): - ''' + """ Fetch a key value. - ''' - c_key = '{0}/{1}'.format(bank, key) + """ + c_key = "{0}/{1}".format(bank, key) try: _, value = api.kv.get(c_key) if value is None: return {} - return __context__['serial'].loads(value['Value']) + return __context__["serial"].loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error reading the key, {0}: {1}'.format( - c_key, exc - ) + "There was an error reading the key, {0}: {1}".format(c_key, exc) ) def flush(bank, key=None): - ''' + """ Remove the key from the cache bank with all the key content. - ''' + """ if key is None: c_key = bank else: - c_key = '{0}/{1}'.format(bank, key) + c_key = "{0}/{1}".format(bank, key) try: return api.kv.delete(c_key, recurse=key is None) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error removing the key, {0}: {1}'.format( - c_key, exc - ) + "There was an error removing the key, {0}: {1}".format(c_key, exc) ) def list_(bank): - ''' + """ Return an iterable object containing all entries stored in the specified bank. - ''' + """ try: - _, keys = api.kv.get(bank + '/', keys=True, separator='/') + _, keys = api.kv.get(bank + "/", keys=True, separator="/") except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error getting the key "{0}": {1}'.format( - bank, exc - ) + 'There was an error getting the key "{0}": {1}'.format(bank, exc) ) if keys is None: keys = [] @@ -163,25 +165,23 @@ def list_(bank): # so we have to return a list of unique names only. out = set() for key in keys: - out.add(key[len(bank) + 1:].rstrip('/')) + out.add(key[len(bank) + 1 :].rstrip("/")) keys = list(out) return keys def contains(bank, key): - ''' + """ Checks if the specified bank contains the specified key. - ''' + """ if key is None: return True # any key could be a branch and a leaf at the same time in Consul else: try: - c_key = '{0}/{1}'.format(bank, key) + c_key = "{0}/{1}".format(bank, key) _, value = api.kv.get(c_key) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error getting the key, {0}: {1}'.format( - c_key, exc - ) + "There was an error getting the key, {0}: {1}".format(c_key, exc) ) return value is not None diff --git a/src/saltext/consul/modules/__init__.py b/src/saltext/consul/modules/__init__.py index 0d649bc..cb0f27f 100644 --- a/src/saltext/consul/modules/__init__.py +++ b/src/saltext/consul/modules/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- -''' +""" Execution Module Directory -''' +""" diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 1b360be..a9f2c2e 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1,19 +1,21 @@ # -*- coding: utf-8 -*- -''' +""" Interact with Consul https://www.consul.io -''' +""" # Import Python Libs from __future__ import absolute_import, print_function, unicode_literals + import base64 import logging # Import salt libs import salt.utils.http import salt.utils.json +from salt.exceptions import SaltInvocationError # Import 3rd-party libs from salt.ext import six @@ -21,40 +23,39 @@ log = logging.getLogger(__name__) -from salt.exceptions import SaltInvocationError # Don't shadow built-ins. -__func_alias__ = { - 'list_': 'list' -} +__func_alias__ = {"list_": "list"} -__virtualname__ = 'consul' +__virtualname__ = "consul" def _get_config(): - ''' + """ Retrieve Consul configuration - ''' - return __salt__['config.get']('consul.url') or \ - __salt__['config.get']('consul:url') + """ + return __salt__["config.get"]("consul.url") or __salt__["config.get"]("consul:url") def _get_token(): - ''' + """ Retrieve Consul configuration - ''' - return __salt__['config.get']('consul.token') or \ - __salt__['config.get']('consul:token') - - -def _query(function, - consul_url, - token=None, - method='GET', - api_version='v1', - data=None, - query_params=None): - ''' + """ + return __salt__["config.get"]("consul.token") or __salt__["config.get"]( + "consul:token" + ) + + +def _query( + function, + consul_url, + token=None, + method="GET", + api_version="v1", + data=None, + query_params=None, +): + """ Consul object method function to construct and execute on the API URL. :param api_url: The Consul api url. @@ -63,22 +64,21 @@ def _query(function, :param method: The HTTP method, e.g. GET or POST. :param data: The data to be sent for POST method. This param is ignored for GET requests. :return: The json response from the API call or False. - ''' + """ if not query_params: query_params = {} - ret = {'data': '', - 'res': True} + ret = {"data": "", "res": True} if not token: token = _get_token() headers = {"X-Consul-Token": token, "Content-Type": "application/json"} - base_url = urllib.parse.urljoin(consul_url, '{0}/'.format(api_version)) + base_url = urllib.parse.urljoin(consul_url, "{0}/".format(api_version)) url = urllib.parse.urljoin(base_url, function, False) - if method == 'GET': + if method == "GET": data = None else: if data is None: @@ -96,25 +96,25 @@ def _query(function, opts=__opts__, ) - if result.get('status', None) == http_client.OK: - ret['data'] = result.get('dict', result) - ret['res'] = True - elif result.get('status', None) == http_client.NO_CONTENT: - ret['res'] = False - elif result.get('status', None) == http_client.NOT_FOUND: - ret['data'] = 'Key not found.' - ret['res'] = False + if result.get("status", None) == http_client.OK: + ret["data"] = result.get("dict", result) + ret["res"] = True + elif result.get("status", None) == http_client.NO_CONTENT: + ret["res"] = False + elif result.get("status", None) == http_client.NOT_FOUND: + ret["data"] = "Key not found." + ret["res"] = False else: if result: - ret['data'] = result - ret['res'] = True + ret["data"] = result + ret["res"] = True else: - ret['res'] = False + ret["res"] = False return ret def list_(consul_url=None, token=None, key=None, **kwargs): - ''' + """ List keys in Consul :param consul_url: The Consul server URL. @@ -128,40 +128,39 @@ def list_(consul_url=None, token=None, key=None, **kwargs): salt '*' consul.list salt '*' consul.list key='web' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret query_params = {} - if 'recurse' in kwargs: - query_params['recurse'] = 'True' + if "recurse" in kwargs: + query_params["recurse"] = "True" # No key so recurse and show all values if not key: - query_params['recurse'] = 'True' - function = 'kv/' + query_params["recurse"] = "True" + function = "kv/" else: - function = 'kv/{0}'.format(key) - - query_params['keys'] = 'True' - query_params['separator'] = '/' - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "kv/{0}".format(key) + + query_params["keys"] = "True" + query_params["separator"] = "/" + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw=False): - ''' + """ Get key from Consul :param consul_url: The Consul server URL. @@ -190,42 +189,41 @@ def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw= By default Consult will return other information about the key, the raw option will return only the raw value. - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not key: raise SaltInvocationError('Required argument "key" is missing.') query_params = {} - function = 'kv/{0}'.format(key) + function = "kv/{0}".format(key) if recurse: - query_params['recurse'] = 'True' + query_params["recurse"] = "True" if raw: - query_params['raw'] = True - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + query_params["raw"] = True + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) - if ret['res']: + if ret["res"]: if decode: - for item in ret['data']: - if item['Value'] is not None: - item['Value'] = base64.b64decode(item['Value']) + for item in ret["data"]: + if item["Value"] is not None: + item["Value"] = base64.b64decode(item["Value"]) else: - item['Value'] = "" + item["Value"] = "" return ret def put(consul_url=None, token=None, key=None, value=None, **kwargs): - ''' + """ Put values into Consul :param consul_url: The Consul server URL. @@ -252,101 +250,108 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): salt '*' consul.put key='web/key1' value="Hello there" release='d5d371f4-c380-5280-12fd-8810be175592' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not key: raise SaltInvocationError('Required argument "key" is missing.') # Invalid to specified these together - conflicting_args = ['cas', 'release', 'acquire'] + conflicting_args = ["cas", "release", "acquire"] for _l1 in conflicting_args: for _l2 in conflicting_args: if _l1 in kwargs and _l2 in kwargs and _l1 != _l2: - raise SaltInvocationError('Using arguments `{0}` and `{1}`' - ' together is invalid.'.format(_l1, _l2)) + raise SaltInvocationError( + "Using arguments `{0}` and `{1}`" + " together is invalid.".format(_l1, _l2) + ) query_params = {} available_sessions = session_list(consul_url=consul_url, return_list=True) _current = get(consul_url=consul_url, key=key) - if 'flags' in kwargs: - if kwargs['flags'] >= 0 and kwargs['flags'] <= 2**64: - query_params['flags'] = kwargs['flags'] - - if 'cas' in kwargs: - if _current['res']: - if kwargs['cas'] == 0: - ret['message'] = ('Key {0} exists, index ' - 'must be non-zero.'.format(key)) - ret['res'] = False + if "flags" in kwargs: + if kwargs["flags"] >= 0 and kwargs["flags"] <= 2 ** 64: + query_params["flags"] = kwargs["flags"] + + if "cas" in kwargs: + if _current["res"]: + if kwargs["cas"] == 0: + ret["message"] = "Key {0} exists, index " "must be non-zero.".format( + key + ) + ret["res"] = False return ret - if kwargs['cas'] != _current['data']['ModifyIndex']: - ret['message'] = ('Key {0} exists, but indexes ' - 'do not match.'.format(key)) - ret['res'] = False + if kwargs["cas"] != _current["data"]["ModifyIndex"]: + ret["message"] = "Key {0} exists, but indexes " "do not match.".format( + key + ) + ret["res"] = False return ret - query_params['cas'] = kwargs['cas'] + query_params["cas"] = kwargs["cas"] else: - ret['message'] = ('Key {0} does not exists, ' - 'CAS argument can not be used.'.format(key)) - ret['res'] = False + ret[ + "message" + ] = "Key {0} does not exists, " "CAS argument can not be used.".format(key) + ret["res"] = False return ret - if 'acquire' in kwargs: - if kwargs['acquire'] not in available_sessions: - ret['message'] = '{0} is not a valid session.'.format(kwargs['acquire']) - ret['res'] = False + if "acquire" in kwargs: + if kwargs["acquire"] not in available_sessions: + ret["message"] = "{0} is not a valid session.".format(kwargs["acquire"]) + ret["res"] = False return ret - query_params['acquire'] = kwargs['acquire'] + query_params["acquire"] = kwargs["acquire"] - if 'release' in kwargs: - if _current['res']: - if 'Session' in _current['data']: - if _current['data']['Session'] == kwargs['release']: - query_params['release'] = kwargs['release'] + if "release" in kwargs: + if _current["res"]: + if "Session" in _current["data"]: + if _current["data"]["Session"] == kwargs["release"]: + query_params["release"] = kwargs["release"] else: - ret['message'] = '{0} locked by another session.'.format(key) - ret['res'] = False + ret["message"] = "{0} locked by another session.".format(key) + ret["res"] = False return ret else: - ret['message'] = '{0} is not a valid session.'.format(kwargs['acquire']) - ret['res'] = False + ret["message"] = "{0} is not a valid session.".format(kwargs["acquire"]) + ret["res"] = False else: - log.error('Key {0} does not exist. Skipping release.') + log.error("Key {0} does not exist. Skipping release.") data = value - function = 'kv/{0}'.format(key) - method = 'PUT' - ret = _query(consul_url=consul_url, - token=token, - function=function, - method=method, - data=data, - query_params=query_params) - - if ret['res']: - ret['res'] = True - ret['data'] = 'Added key {0} with value {1}.'.format(key, value) + function = "kv/{0}".format(key) + method = "PUT" + ret = _query( + consul_url=consul_url, + token=token, + function=function, + method=method, + data=data, + query_params=query_params, + ) + + if ret["res"]: + ret["res"] = True + ret["data"] = "Added key {0} with value {1}.".format(key, value) else: - ret['res'] = False - ret['data'] = 'Unable to add key {0} with value {1}.'.format(key, value) + ret["res"] = False + ret["data"] = "Unable to add key {0} with value {1}.".format(key, value) return ret def delete(consul_url=None, token=None, key=None, **kwargs): - ''' + """ Delete values from Consul :param consul_url: The Consul server URL. @@ -363,14 +368,14 @@ def delete(consul_url=None, token=None, key=None, **kwargs): salt '*' consul.delete key='web' salt '*' consul.delete key='web' recurse='True' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not key: @@ -378,36 +383,40 @@ def delete(consul_url=None, token=None, key=None, **kwargs): query_params = {} - if 'recurse' in kwargs: - query_params['recurse'] = True + if "recurse" in kwargs: + query_params["recurse"] = True - if 'cas' in kwargs: - if kwargs['cas'] > 0: - query_params['cas'] = kwargs['cas'] + if "cas" in kwargs: + if kwargs["cas"] > 0: + query_params["cas"] = kwargs["cas"] else: - ret['message'] = ('Check and Set Operation ', - 'value must be greater than 0.') - ret['res'] = False + ret["message"] = ( + "Check and Set Operation ", + "value must be greater than 0.", + ) + ret["res"] = False return ret - function = 'kv/{0}'.format(key) - ret = _query(consul_url=consul_url, - token=token, - function=function, - method='DELETE', - query_params=query_params) + function = "kv/{0}".format(key) + ret = _query( + consul_url=consul_url, + token=token, + function=function, + method="DELETE", + query_params=query_params, + ) - if ret['res']: - ret['res'] = True - ret['message'] = 'Deleted key {0}.'.format(key) + if ret["res"]: + ret["res"] = True + ret["message"] = "Deleted key {0}.".format(key) else: - ret['res'] = False - ret['message'] = 'Unable to delete key {0}.'.format(key) + ret["res"] = False + ret["message"] = "Unable to delete key {0}.".format(key) return ret def agent_checks(consul_url=None, token=None): - ''' + """ Returns the checks the local agent is managing :param consul_url: The Consul server URL. @@ -419,26 +428,23 @@ def agent_checks(consul_url=None, token=None): salt '*' consul.agent_checks - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'agent/checks' - ret = _query(consul_url=consul_url, - function=function, - token=token, - method='GET') + function = "agent/checks" + ret = _query(consul_url=consul_url, function=function, token=token, method="GET") return ret def agent_services(consul_url=None, token=None): - ''' + """ Returns the services the local agent is managing :param consul_url: The Consul server URL. @@ -450,26 +456,23 @@ def agent_services(consul_url=None, token=None): salt '*' consul.agent_services - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'agent/services' - ret = _query(consul_url=consul_url, - function=function, - token=token, - method='GET') + function = "agent/services" + ret = _query(consul_url=consul_url, function=function, token=token, method="GET") return ret def agent_members(consul_url=None, token=None, **kwargs): - ''' + """ Returns the members as seen by the local serf agent :param consul_url: The Consul server URL. @@ -481,31 +484,33 @@ def agent_members(consul_url=None, token=None, **kwargs): salt '*' consul.agent_members - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'wan' in kwargs: - query_params['wan'] = kwargs['wan'] + if "wan" in kwargs: + query_params["wan"] = kwargs["wan"] - function = 'agent/members' - ret = _query(consul_url=consul_url, - function=function, - token=token, - method='GET', - query_params=query_params) + function = "agent/members" + ret = _query( + consul_url=consul_url, + function=function, + token=token, + method="GET", + query_params=query_params, + ) return ret def agent_self(consul_url=None, token=None): - ''' + """ Returns the local node configuration :param consul_url: The Consul server URL. @@ -517,28 +522,30 @@ def agent_self(consul_url=None, token=None): salt '*' consul.agent_self - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'agent/self' - ret = _query(consul_url=consul_url, - function=function, - token=token, - method='GET', - query_params=query_params) + function = "agent/self" + ret = _query( + consul_url=consul_url, + function=function, + token=token, + method="GET", + query_params=query_params, + ) return ret def agent_maintenance(consul_url=None, token=None, **kwargs): - ''' + """ Manages node maintenance mode :param consul_url: The Consul server URL. @@ -557,45 +564,46 @@ def agent_maintenance(consul_url=None, token=None, **kwargs): salt '*' consul.agent_maintenance enable='False' reason='Upgrade in progress' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'enable' in kwargs: - query_params['enable'] = kwargs['enable'] + if "enable" in kwargs: + query_params["enable"] = kwargs["enable"] else: - ret['message'] = 'Required parameter "enable" is missing.' - ret['res'] = False + ret["message"] = 'Required parameter "enable" is missing.' + ret["res"] = False return ret - if 'reason' in kwargs: - query_params['reason'] = kwargs['reason'] - - function = 'agent/maintenance' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - query_params=query_params) - if res['res']: - ret['res'] = True - ret['message'] = ('Agent maintenance mode ' - '{0}ed.'.format(kwargs['enable'])) + if "reason" in kwargs: + query_params["reason"] = kwargs["reason"] + + function = "agent/maintenance" + res = _query( + consul_url=consul_url, + function=function, + token=token, + method="PUT", + query_params=query_params, + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Agent maintenance mode " "{0}ed.".format(kwargs["enable"]) else: - ret['res'] = True - ret['message'] = 'Unable to change maintenance mode for agent.' + ret["res"] = True + ret["message"] = "Unable to change maintenance mode for agent." return ret def agent_join(consul_url=None, token=None, address=None, **kwargs): - ''' + """ Triggers the local agent to join a node :param consul_url: The Consul server URL. @@ -609,40 +617,42 @@ def agent_join(consul_url=None, token=None, address=None, **kwargs): salt '*' consul.agent_join address='192.168.1.1' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not address: raise SaltInvocationError('Required argument "address" is missing.') - if 'wan' in kwargs: - query_params['wan'] = kwargs['wan'] - - function = 'agent/join/{0}'.format(address) - res = _query(consul_url=consul_url, - function=function, - token=token, - method='GET', - query_params=query_params) - if res['res']: - ret['res'] = True - ret['message'] = 'Agent joined the cluster' + if "wan" in kwargs: + query_params["wan"] = kwargs["wan"] + + function = "agent/join/{0}".format(address) + res = _query( + consul_url=consul_url, + function=function, + token=token, + method="GET", + query_params=query_params, + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Agent joined the cluster" else: - ret['res'] = False - ret['message'] = 'Unable to join the cluster.' + ret["res"] = False + ret["message"] = "Unable to join the cluster." return ret def agent_leave(consul_url=None, token=None, node=None): - ''' + """ Used to instruct the agent to force a node into the left state. :param consul_url: The Consul server URL. @@ -655,37 +665,39 @@ def agent_leave(consul_url=None, token=None, node=None): salt '*' consul.agent_leave node='web1.example.com' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not node: raise SaltInvocationError('Required argument "node" is missing.') - function = 'agent/force-leave/{0}'.format(node) - res = _query(consul_url=consul_url, - function=function, - token=token, - method='GET', - query_params=query_params) - if res['res']: - ret['res'] = True - ret['message'] = 'Node {0} put in leave state.'.format(node) + function = "agent/force-leave/{0}".format(node) + res = _query( + consul_url=consul_url, + function=function, + token=token, + method="GET", + query_params=query_params, + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Node {0} put in leave state.".format(node) else: - ret['res'] = False - ret['message'] = 'Unable to change state for {0}.'.format(node) + ret["res"] = False + ret["message"] = "Unable to change state for {0}.".format(node) return ret def agent_check_register(consul_url=None, token=None, **kwargs): - ''' + """ The register endpoint is used to add a new check to the local agent. :param consul_url: The Consul server URL. @@ -710,70 +722,68 @@ def agent_check_register(consul_url=None, token=None, **kwargs): salt '*' consul.agent_check_register name='Memory Utilization' script='/usr/local/bin/check_mem.py' interval='15s' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'name' in kwargs: - data['Name'] = kwargs['name'] + if "name" in kwargs: + data["Name"] = kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - if True not in [True for item in ('script', 'http', 'ttl') if item in kwargs]: - ret['message'] = 'Required parameter "script" or "http" is missing.' - ret['res'] = False + if True not in [True for item in ("script", "http", "ttl") if item in kwargs]: + ret["message"] = 'Required parameter "script" or "http" is missing.' + ret["res"] = False return ret - if 'id' in kwargs: - data['ID'] = kwargs['id'] + if "id" in kwargs: + data["ID"] = kwargs["id"] - if 'notes' in kwargs: - data['Notes'] = kwargs['notes'] + if "notes" in kwargs: + data["Notes"] = kwargs["notes"] - if 'script' in kwargs: - if 'interval' not in kwargs: - ret['message'] = 'Required parameter "interval" is missing.' - ret['res'] = False + if "script" in kwargs: + if "interval" not in kwargs: + ret["message"] = 'Required parameter "interval" is missing.' + ret["res"] = False return ret - data['Script'] = kwargs['script'] - data['Interval'] = kwargs['interval'] + data["Script"] = kwargs["script"] + data["Interval"] = kwargs["interval"] - if 'http' in kwargs: - if 'interval' not in kwargs: - ret['message'] = 'Required parameter "interval" is missing.' - ret['res'] = False + if "http" in kwargs: + if "interval" not in kwargs: + ret["message"] = 'Required parameter "interval" is missing.' + ret["res"] = False return ret - data['HTTP'] = kwargs['http'] - data['Interval'] = kwargs['interval'] - - if 'ttl' in kwargs: - data['TTL'] = kwargs['ttl'] - - function = 'agent/check/register' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) - - if res['res']: - ret['res'] = True - ret['message'] = ('Check {0} added to agent.'.format(kwargs['name'])) + data["HTTP"] = kwargs["http"] + data["Interval"] = kwargs["interval"] + + if "ttl" in kwargs: + data["TTL"] = kwargs["ttl"] + + function = "agent/check/register" + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) + + if res["res"]: + ret["res"] = True + ret["message"] = "Check {0} added to agent.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = 'Unable to add check to agent.' + ret["res"] = False + ret["message"] = "Unable to add check to agent." return ret def agent_check_deregister(consul_url=None, token=None, checkid=None): - ''' + """ The agent will take care of deregistering the check from the Catalog. :param consul_url: The Consul server URL. @@ -786,35 +796,32 @@ def agent_check_deregister(consul_url=None, token=None, checkid=None): salt '*' consul.agent_check_deregister checkid='Memory Utilization' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - function = 'agent/check/deregister/{0}'.format(checkid) - res = _query(consul_url=consul_url, - function=function, - token=token, - method='GET') - if res['res']: - ret['res'] = True - ret['message'] = ('Check {0} removed from agent.'.format(checkid)) + function = "agent/check/deregister/{0}".format(checkid) + res = _query(consul_url=consul_url, function=function, token=token, method="GET") + if res["res"]: + ret["res"] = True + ret["message"] = "Check {0} removed from agent.".format(checkid) else: - ret['res'] = False - ret['message'] = 'Unable to remove check from agent.' + ret["res"] = False + ret["message"] = "Unable to remove check from agent." return ret def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): - ''' + """ This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to passing and the TTL clock is reset. @@ -830,40 +837,42 @@ def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): salt '*' consul.agent_check_pass checkid='redis_check1' note='Forcing check into passing state.' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - if 'note' in kwargs: - query_params['note'] = kwargs['note'] - - function = 'agent/check/pass/{0}'.format(checkid) - res = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params, - method='GET') - if res['res']: - ret['res'] = True - ret['message'] = 'Check {0} marked as passing.'.format(checkid) + if "note" in kwargs: + query_params["note"] = kwargs["note"] + + function = "agent/check/pass/{0}".format(checkid) + res = _query( + consul_url=consul_url, + function=function, + token=token, + query_params=query_params, + method="GET", + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Check {0} marked as passing.".format(checkid) else: - ret['res'] = False - ret['message'] = 'Unable to update check {0}.'.format(checkid) + ret["res"] = False + ret["message"] = "Unable to update check {0}.".format(checkid) return ret def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): - ''' + """ This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to warning and the TTL clock is reset. @@ -879,40 +888,42 @@ def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): salt '*' consul.agent_check_warn checkid='redis_check1' note='Forcing check into warning state.' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - if 'note' in kwargs: - query_params['note'] = kwargs['note'] - - function = 'agent/check/warn/{0}'.format(checkid) - res = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params, - method='GET') - if res['res']: - ret['res'] = True - ret['message'] = 'Check {0} marked as warning.'.format(checkid) + if "note" in kwargs: + query_params["note"] = kwargs["note"] + + function = "agent/check/warn/{0}".format(checkid) + res = _query( + consul_url=consul_url, + function=function, + token=token, + query_params=query_params, + method="GET", + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Check {0} marked as warning.".format(checkid) else: - ret['res'] = False - ret['message'] = 'Unable to update check {0}.'.format(checkid) + ret["res"] = False + ret["message"] = "Unable to update check {0}.".format(checkid) return ret def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): - ''' + """ This endpoint is used with a check that is of the TTL type. When this is called, the status of the check is set to critical and the TTL clock is reset. @@ -928,40 +939,42 @@ def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): salt '*' consul.agent_check_fail checkid='redis_check1' note='Forcing check into critical state.' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - if 'note' in kwargs: - query_params['note'] = kwargs['note'] - - function = 'agent/check/fail/{0}'.format(checkid) - res = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params, - method='GET') - if res['res']: - ret['res'] = True - ret['message'] = 'Check {0} marked as critical.'.format(checkid) + if "note" in kwargs: + query_params["note"] = kwargs["note"] + + function = "agent/check/fail/{0}".format(checkid) + res = _query( + consul_url=consul_url, + function=function, + token=token, + query_params=query_params, + method="GET", + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Check {0} marked as critical.".format(checkid) else: - ret['res'] = False - ret['message'] = 'Unable to update check {0}.'.format(checkid) + ret["res"] = False + ret["message"] = "Unable to update check {0}.".format(checkid) return ret def agent_service_register(consul_url=None, token=None, **kwargs): - ''' + """ The used to add a new service, with an optional health check, to the local agent. @@ -991,91 +1004,89 @@ def agent_service_register(consul_url=None, token=None, **kwargs): salt '*' consul.agent_service_register name='redis' tags='["master", "v1"]' address="127.0.0.1" port="8080" check_script="/usr/local/bin/check_redis.py" interval="10s" - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret lc_kwargs = dict() for k, v in six.iteritems(kwargs): lc_kwargs[k.lower()] = v - if 'name' in lc_kwargs: - data['Name'] = lc_kwargs['name'] + if "name" in lc_kwargs: + data["Name"] = lc_kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - if 'address' in lc_kwargs: - data['Address'] = lc_kwargs['address'] + if "address" in lc_kwargs: + data["Address"] = lc_kwargs["address"] - if 'port' in lc_kwargs: - data['Port'] = lc_kwargs['port'] + if "port" in lc_kwargs: + data["Port"] = lc_kwargs["port"] - if 'id' in lc_kwargs: - data['ID'] = lc_kwargs['id'] + if "id" in lc_kwargs: + data["ID"] = lc_kwargs["id"] - if 'tags' in lc_kwargs: - _tags = lc_kwargs['tags'] + if "tags" in lc_kwargs: + _tags = lc_kwargs["tags"] if not isinstance(_tags, list): _tags = [_tags] - data['Tags'] = _tags + data["Tags"] = _tags - if 'enabletagoverride' in lc_kwargs: - data['EnableTagOverride'] = lc_kwargs['enabletagoverride'] + if "enabletagoverride" in lc_kwargs: + data["EnableTagOverride"] = lc_kwargs["enabletagoverride"] - if 'check' in lc_kwargs: + if "check" in lc_kwargs: dd = dict() - for k, v in six.iteritems(lc_kwargs['check']): + for k, v in six.iteritems(lc_kwargs["check"]): dd[k.lower()] = v interval_required = False check_dd = dict() - if 'script' in dd: + if "script" in dd: interval_required = True - check_dd['Script'] = dd['script'] - if 'http' in dd: + check_dd["Script"] = dd["script"] + if "http" in dd: interval_required = True - check_dd['HTTP'] = dd['http'] - if 'ttl' in dd: - check_dd['TTL'] = dd['ttl'] - if 'interval' in dd: - check_dd['Interval'] = dd['interval'] + check_dd["HTTP"] = dd["http"] + if "ttl" in dd: + check_dd["TTL"] = dd["ttl"] + if "interval" in dd: + check_dd["Interval"] = dd["interval"] if interval_required: - if 'Interval' not in check_dd: - ret['message'] = 'Required parameter "interval" is missing.' - ret['res'] = False + if "Interval" not in check_dd: + ret["message"] = 'Required parameter "interval" is missing.' + ret["res"] = False return ret else: - if 'Interval' in check_dd: - del check_dd['Interval'] # not required, so ignore it + if "Interval" in check_dd: + del check_dd["Interval"] # not required, so ignore it if check_dd > 0: - data['Check'] = check_dd # if empty, ignore it - - function = 'agent/service/register' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) - if res['res']: - ret['res'] = True - ret['message'] = 'Service {0} registered on agent.'.format(kwargs['name']) + data["Check"] = check_dd # if empty, ignore it + + function = "agent/service/register" + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Service {0} registered on agent.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = 'Unable to register service {0}.'.format(kwargs['name']) + ret["res"] = False + ret["message"] = "Unable to register service {0}.".format(kwargs["name"]) return ret def agent_service_deregister(consul_url=None, token=None, serviceid=None): - ''' + """ Used to remove a service. :param consul_url: The Consul server URL. @@ -1088,37 +1099,35 @@ def agent_service_deregister(consul_url=None, token=None, serviceid=None): salt '*' consul.agent_service_deregister serviceid='redis' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not serviceid: raise SaltInvocationError('Required argument "serviceid" is missing.') - function = 'agent/service/deregister/{0}'.format(serviceid) - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) - if res['res']: - ret['res'] = True - ret['message'] = 'Service {0} removed from agent.'.format(serviceid) + function = "agent/service/deregister/{0}".format(serviceid) + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Service {0} removed from agent.".format(serviceid) else: - ret['res'] = False - ret['message'] = 'Unable to remove service {0}.'.format(serviceid) + ret["res"] = False + ret["message"] = "Unable to remove service {0}.".format(serviceid) return ret def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwargs): - ''' + """ Used to place a service into maintenance mode. :param consul_url: The Consul server URL. @@ -1134,49 +1143,48 @@ def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwa salt '*' consul.agent_service_deregister serviceid='redis' enable='True' reason='Down for upgrade' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not serviceid: raise SaltInvocationError('Required argument "serviceid" is missing.') - if 'enable' in kwargs: - query_params['enable'] = kwargs['enable'] + if "enable" in kwargs: + query_params["enable"] = kwargs["enable"] else: - ret['message'] = 'Required parameter "enable" is missing.' - ret['res'] = False + ret["message"] = 'Required parameter "enable" is missing.' + ret["res"] = False return ret - if 'reason' in kwargs: - query_params['reason'] = kwargs['reason'] + if "reason" in kwargs: + query_params["reason"] = kwargs["reason"] - function = 'agent/service/maintenance/{0}'.format(serviceid) - res = _query(consul_url=consul_url, - token=token, - function=function, - query_params=query_params) + function = "agent/service/maintenance/{0}".format(serviceid) + res = _query( + consul_url=consul_url, token=token, function=function, query_params=query_params + ) - if res['res']: - ret['res'] = True - ret['message'] = ('Service {0} set in ' - 'maintenance mode.'.format(serviceid)) + if res["res"]: + ret["res"] = True + ret["message"] = "Service {0} set in " "maintenance mode.".format(serviceid) else: - ret['res'] = False - ret['message'] = ('Unable to set service ' - '{0} to maintenance mode.'.format(serviceid)) + ret["res"] = False + ret["message"] = "Unable to set service " "{0} to maintenance mode.".format( + serviceid + ) return ret def session_create(consul_url=None, token=None, **kwargs): - ''' + """ Used to create a session. :param consul_url: The Consul server URL. @@ -1205,69 +1213,65 @@ def session_create(consul_url=None, token=None, **kwargs): salt '*' consul.session_create node='node1' name='my-session' behavior='delete' ttl='3600s' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret data = {} - if 'lockdelay' in kwargs: - data['LockDelay'] = kwargs['lockdelay'] + if "lockdelay" in kwargs: + data["LockDelay"] = kwargs["lockdelay"] - if 'node' in kwargs: - data['Node'] = kwargs['node'] + if "node" in kwargs: + data["Node"] = kwargs["node"] - if 'name' in kwargs: - data['Name'] = kwargs['name'] + if "name" in kwargs: + data["Name"] = kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - if 'checks' in kwargs: - data['Touch'] = kwargs['touch'] + if "checks" in kwargs: + data["Touch"] = kwargs["touch"] - if 'behavior' in kwargs: - if not kwargs['behavior'] in ('delete', 'release'): - ret['message'] = ('Behavior must be ', - 'either delete or release.') - ret['res'] = False + if "behavior" in kwargs: + if not kwargs["behavior"] in ("delete", "release"): + ret["message"] = ("Behavior must be ", "either delete or release.") + ret["res"] = False return ret - data['Behavior'] = kwargs['behavior'] + data["Behavior"] = kwargs["behavior"] - if 'ttl' in kwargs: - _ttl = kwargs['ttl'] - if six.text_type(_ttl).endswith('s'): + if "ttl" in kwargs: + _ttl = kwargs["ttl"] + if six.text_type(_ttl).endswith("s"): _ttl = _ttl[:-1] if int(_ttl) < 0 or int(_ttl) > 3600: - ret['message'] = ('TTL must be ', - 'between 0 and 3600.') - ret['res'] = False + ret["message"] = ("TTL must be ", "between 0 and 3600.") + ret["res"] = False return ret - data['TTL'] = '{0}s'.format(_ttl) - - function = 'session/create' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) - - if res['res']: - ret['res'] = True - ret['message'] = 'Created session {0}.'.format(kwargs['name']) + data["TTL"] = "{0}s".format(_ttl) + + function = "session/create" + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) + + if res["res"]: + ret["res"] = True + ret["message"] = "Created session {0}.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = 'Unable to create session {0}.'.format(kwargs['name']) + ret["res"] = False + ret["message"] = "Unable to create session {0}.".format(kwargs["name"]) return ret def session_list(consul_url=None, token=None, return_list=False, **kwargs): - ''' + """ Used to list sessions. :param consul_url: The Consul server URL. @@ -1284,37 +1288,36 @@ def session_list(consul_url=None, token=None, return_list=False, **kwargs): salt '*' consul.session_list - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret query_params = {} - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'session/list' - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "session/list" + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) if return_list: _list = [] - for item in ret['data']: - _list.append(item['ID']) + for item in ret["data"]: + _list.append(item["ID"]) return _list return ret def session_destroy(consul_url=None, token=None, session=None, **kwargs): - ''' + """ Destroy session :param consul_url: The Consul server URL. @@ -1329,14 +1332,14 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): salt '*' consul.session_destroy session='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not session: @@ -1344,25 +1347,24 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): query_params = {} - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] - - function = 'session/destroy/{0}'.format(session) - res = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) - if res['res']: - ret['res'] = True - ret['message'] = 'Created Service {0}.'.format(kwargs['name']) + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] + + function = "session/destroy/{0}".format(session) + res = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Created Service {0}.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = 'Unable to create service {0}.'.format(kwargs['name']) + ret["res"] = False + ret["message"] = "Unable to create service {0}.".format(kwargs["name"]) return ret def session_info(consul_url=None, token=None, session=None, **kwargs): - ''' + """ Information about a session :param consul_url: The Consul server URL. @@ -1377,14 +1379,14 @@ def session_info(consul_url=None, token=None, session=None, **kwargs): salt '*' consul.session_info session='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not session: @@ -1392,19 +1394,18 @@ def session_info(consul_url=None, token=None, session=None, **kwargs): query_params = {} - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'session/info/{0}'.format(session) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "session/info/{0}".format(session) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def catalog_register(consul_url=None, token=None, **kwargs): - ''' + """ Registers a new node, service, or check :param consul_url: The Consul server URL. @@ -1432,140 +1433,147 @@ def catalog_register(consul_url=None, token=None, **kwargs): salt '*' consul.catalog_register node='node1' address='192.168.1.1' service='redis' service_address='127.0.0.1' service_port='8080' service_id='redis_server1' - ''' + """ ret = {} data = {} - data['NodeMeta'] = {} + data["NodeMeta"] = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'datacenter' in kwargs: - data['Datacenter'] = kwargs['datacenter'] + if "datacenter" in kwargs: + data["Datacenter"] = kwargs["datacenter"] - if 'node' in kwargs: - data['Node'] = kwargs['node'] + if "node" in kwargs: + data["Node"] = kwargs["node"] else: - ret['message'] = 'Required argument node argument is missing.' - ret['res'] = False + ret["message"] = "Required argument node argument is missing." + ret["res"] = False return ret - if 'address' in kwargs: - if isinstance(kwargs['address'], list): - _address = kwargs['address'][0] + if "address" in kwargs: + if isinstance(kwargs["address"], list): + _address = kwargs["address"][0] else: - _address = kwargs['address'] - data['Address'] = _address + _address = kwargs["address"] + data["Address"] = _address else: - ret['message'] = 'Required argument address argument is missing.' - ret['res'] = False + ret["message"] = "Required argument address argument is missing." + ret["res"] = False return ret - if 'ip_interfaces' in kwargs: - data['TaggedAddresses'] = {} - for k in kwargs['ip_interfaces']: - if kwargs['ip_interfaces'].get(k): - data['TaggedAddresses'][k] = kwargs['ip_interfaces'][k][0] + if "ip_interfaces" in kwargs: + data["TaggedAddresses"] = {} + for k in kwargs["ip_interfaces"]: + if kwargs["ip_interfaces"].get(k): + data["TaggedAddresses"][k] = kwargs["ip_interfaces"][k][0] - if 'service' in kwargs: - data['Service'] = {} - data['Service']['Service'] = kwargs['service'] + if "service" in kwargs: + data["Service"] = {} + data["Service"]["Service"] = kwargs["service"] - if 'service_address' in kwargs: - data['Service']['Address'] = kwargs['service_address'] + if "service_address" in kwargs: + data["Service"]["Address"] = kwargs["service_address"] - if 'service_port' in kwargs: - data['Service']['Port'] = kwargs['service_port'] + if "service_port" in kwargs: + data["Service"]["Port"] = kwargs["service_port"] - if 'service_id' in kwargs: - data['Service']['ID'] = kwargs['service_id'] + if "service_id" in kwargs: + data["Service"]["ID"] = kwargs["service_id"] - if 'service_tags' in kwargs: - _tags = kwargs['service_tags'] + if "service_tags" in kwargs: + _tags = kwargs["service_tags"] if not isinstance(_tags, list): _tags = [_tags] - data['Service']['Tags'] = _tags + data["Service"]["Tags"] = _tags - if 'cpu' in kwargs: - data['NodeMeta']['Cpu'] = kwargs['cpu'] + if "cpu" in kwargs: + data["NodeMeta"]["Cpu"] = kwargs["cpu"] - if 'num_cpus' in kwargs: - data['NodeMeta']['Cpu_num'] = kwargs['num_cpus'] + if "num_cpus" in kwargs: + data["NodeMeta"]["Cpu_num"] = kwargs["num_cpus"] - if 'mem' in kwargs: - data['NodeMeta']['Memory'] = kwargs['mem'] + if "mem" in kwargs: + data["NodeMeta"]["Memory"] = kwargs["mem"] - if 'oscode' in kwargs: - data['NodeMeta']['Os'] = kwargs['oscode'] + if "oscode" in kwargs: + data["NodeMeta"]["Os"] = kwargs["oscode"] - if 'osarch' in kwargs: - data['NodeMeta']['Osarch'] = kwargs['osarch'] + if "osarch" in kwargs: + data["NodeMeta"]["Osarch"] = kwargs["osarch"] - if 'kernel' in kwargs: - data['NodeMeta']['Kernel'] = kwargs['kernel'] + if "kernel" in kwargs: + data["NodeMeta"]["Kernel"] = kwargs["kernel"] - if 'kernelrelease' in kwargs: - data['NodeMeta']['Kernelrelease'] = kwargs['kernelrelease'] + if "kernelrelease" in kwargs: + data["NodeMeta"]["Kernelrelease"] = kwargs["kernelrelease"] - if 'localhost' in kwargs: - data['NodeMeta']['localhost'] = kwargs['localhost'] + if "localhost" in kwargs: + data["NodeMeta"]["localhost"] = kwargs["localhost"] - if 'nodename' in kwargs: - data['NodeMeta']['nodename'] = kwargs['nodename'] + if "nodename" in kwargs: + data["NodeMeta"]["nodename"] = kwargs["nodename"] - if 'os_family' in kwargs: - data['NodeMeta']['os_family'] = kwargs['os_family'] + if "os_family" in kwargs: + data["NodeMeta"]["os_family"] = kwargs["os_family"] - if 'lsb_distrib_description' in kwargs: - data['NodeMeta']['lsb_distrib_description'] = kwargs['lsb_distrib_description'] + if "lsb_distrib_description" in kwargs: + data["NodeMeta"]["lsb_distrib_description"] = kwargs["lsb_distrib_description"] - if 'master' in kwargs: - data['NodeMeta']['master'] = kwargs['master'] + if "master" in kwargs: + data["NodeMeta"]["master"] = kwargs["master"] - if 'check' in kwargs: - data['Check'] = {} - data['Check']['Name'] = kwargs['check'] + if "check" in kwargs: + data["Check"] = {} + data["Check"]["Name"] = kwargs["check"] - if 'check_status' in kwargs: - if kwargs['check_status'] not in ('unknown', 'passing', 'warning', 'critical'): - ret['message'] = 'Check status must be unknown, passing, warning, or critical.' - ret['res'] = False + if "check_status" in kwargs: + if kwargs["check_status"] not in ( + "unknown", + "passing", + "warning", + "critical", + ): + ret[ + "message" + ] = "Check status must be unknown, passing, warning, or critical." + ret["res"] = False return ret - data['Check']['Status'] = kwargs['check_status'] - - if 'check_service' in kwargs: - data['Check']['ServiceID'] = kwargs['check_service'] - - if 'check_id' in kwargs: - data['Check']['CheckID'] = kwargs['check_id'] - - if 'check_notes' in kwargs: - data['Check']['Notes'] = kwargs['check_notes'] - - function = 'catalog/register' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) - if res['res']: - ret['res'] = True - ret['message'] = ('Catalog registration ' - 'for {0} successful.'.format(kwargs['node'])) + data["Check"]["Status"] = kwargs["check_status"] + + if "check_service" in kwargs: + data["Check"]["ServiceID"] = kwargs["check_service"] + + if "check_id" in kwargs: + data["Check"]["CheckID"] = kwargs["check_id"] + + if "check_notes" in kwargs: + data["Check"]["Notes"] = kwargs["check_notes"] + + function = "catalog/register" + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) + if res["res"]: + ret["res"] = True + ret["message"] = "Catalog registration " "for {0} successful.".format( + kwargs["node"] + ) else: - ret['res'] = False - ret['message'] = ('Catalog registration ' - 'for {0} failed.'.format(kwargs['node'])) - ret['data'] = data + ret["res"] = False + ret["message"] = "Catalog registration " "for {0} failed.".format( + kwargs["node"] + ) + ret["data"] = data return ret def catalog_deregister(consul_url=None, token=None, **kwargs): - ''' + """ Deregisters a node, service, or check :param consul_url: The Consul server URL. @@ -1582,52 +1590,49 @@ def catalog_deregister(consul_url=None, token=None, **kwargs): salt '*' consul.catalog_register node='node1' serviceid='redis_server1' checkid='redis_check1' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'datacenter' in kwargs: - data['Datacenter'] = kwargs['datacenter'] + if "datacenter" in kwargs: + data["Datacenter"] = kwargs["datacenter"] - if 'node' in kwargs: - data['Node'] = kwargs['node'] + if "node" in kwargs: + data["Node"] = kwargs["node"] else: - ret['message'] = 'Node argument required.' - ret['res'] = False + ret["message"] = "Node argument required." + ret["res"] = False return ret - if 'checkid' in kwargs: - data['CheckID'] = kwargs['checkid'] + if "checkid" in kwargs: + data["CheckID"] = kwargs["checkid"] - if 'serviceid' in kwargs: - data['ServiceID'] = kwargs['serviceid'] + if "serviceid" in kwargs: + data["ServiceID"] = kwargs["serviceid"] - function = 'catalog/deregister' - res = _query(consul_url=consul_url, - function=function, - token=token, - method='PUT', - data=data) + function = "catalog/deregister" + res = _query( + consul_url=consul_url, function=function, token=token, method="PUT", data=data + ) - if res['res']: - ret['res'] = True - ret['message'] = 'Catalog item {0} removed.'.format(kwargs['node']) + if res["res"]: + ret["res"] = True + ret["message"] = "Catalog item {0} removed.".format(kwargs["node"]) else: - ret['res'] = False - ret['message'] = ('Removing Catalog ' - 'item {0} failed.'.format(kwargs['node'])) + ret["res"] = False + ret["message"] = "Removing Catalog " "item {0} failed.".format(kwargs["node"]) return ret def catalog_datacenters(consul_url=None, token=None): - ''' + """ Return list of available datacenters from catalog. :param consul_url: The Consul server URL. @@ -1639,25 +1644,23 @@ def catalog_datacenters(consul_url=None, token=None): salt '*' consul.catalog_datacenters - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'catalog/datacenters' - ret = _query(consul_url=consul_url, - function=function, - token=token) + function = "catalog/datacenters" + ret = _query(consul_url=consul_url, function=function, token=token) return ret def catalog_nodes(consul_url=None, token=None, **kwargs): - ''' + """ Return list of available nodes from catalog. :param consul_url: The Consul server URL. @@ -1671,30 +1674,29 @@ def catalog_nodes(consul_url=None, token=None, **kwargs): salt '*' consul.catalog_nodes - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'catalog/nodes' - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "catalog/nodes" + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def catalog_services(consul_url=None, token=None, **kwargs): - ''' + """ Return list of available services rom catalog. :param consul_url: The Consul server URL. @@ -1708,30 +1710,29 @@ def catalog_services(consul_url=None, token=None, **kwargs): salt '*' consul.catalog_services - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'catalog/services' - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "catalog/services" + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def catalog_service(consul_url=None, token=None, service=None, **kwargs): - ''' + """ Information about the registered service. :param consul_url: The Consul server URL. @@ -1746,36 +1747,35 @@ def catalog_service(consul_url=None, token=None, service=None, **kwargs): salt '*' consul.catalog_service service='redis' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not service: raise SaltInvocationError('Required argument "service" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - if 'tag' in kwargs: - query_params['tag'] = kwargs['tag'] + if "tag" in kwargs: + query_params["tag"] = kwargs["tag"] - function = 'catalog/service/{0}'.format(service) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "catalog/service/{0}".format(service) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def catalog_node(consul_url=None, token=None, node=None, **kwargs): - ''' + """ Information about the registered node. :param consul_url: The Consul server URL. @@ -1790,33 +1790,32 @@ def catalog_node(consul_url=None, token=None, node=None, **kwargs): salt '*' consul.catalog_service service='redis' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not node: raise SaltInvocationError('Required argument "node" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'catalog/node/{0}'.format(node) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "catalog/node/{0}".format(node) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def health_node(consul_url=None, token=None, node=None, **kwargs): - ''' + """ Health information about the registered node. :param consul_url: The Consul server URL. @@ -1831,33 +1830,32 @@ def health_node(consul_url=None, token=None, node=None, **kwargs): salt '*' consul.health_node node='node1' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not node: raise SaltInvocationError('Required argument "node" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'health/node/{0}'.format(node) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "health/node/{0}".format(node) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def health_checks(consul_url=None, token=None, service=None, **kwargs): - ''' + """ Health information about the registered service. :param consul_url: The Consul server URL. @@ -1872,33 +1870,32 @@ def health_checks(consul_url=None, token=None, service=None, **kwargs): salt '*' consul.health_checks service='redis1' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not service: raise SaltInvocationError('Required argument "service" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - function = 'health/checks/{0}'.format(service) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "health/checks/{0}".format(service) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def health_service(consul_url=None, token=None, service=None, **kwargs): - ''' + """ Health information about the registered service. :param consul_url: The Consul server URL. @@ -1918,39 +1915,38 @@ def health_service(consul_url=None, token=None, service=None, **kwargs): salt '*' consul.health_service service='redis1' passing='True' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not service: raise SaltInvocationError('Required argument "service" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - if 'tag' in kwargs: - query_params['tag'] = kwargs['tag'] + if "tag" in kwargs: + query_params["tag"] = kwargs["tag"] - if 'passing' in kwargs: - query_params['passing'] = kwargs['passing'] + if "passing" in kwargs: + query_params["passing"] = kwargs["passing"] - function = 'health/service/{0}'.format(service) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "health/service/{0}".format(service) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def health_state(consul_url=None, token=None, state=None, **kwargs): - ''' + """ Returns the checks in the state provided on the path. :param consul_url: The Consul server URL. @@ -1970,38 +1966,37 @@ def health_state(consul_url=None, token=None, state=None, **kwargs): salt '*' consul.health_state service='redis1' passing='True' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not state: raise SaltInvocationError('Required argument "state" is missing.') - if 'dc' in kwargs: - query_params['dc'] = kwargs['dc'] + if "dc" in kwargs: + query_params["dc"] = kwargs["dc"] - if state not in ('any', 'unknown', 'passing', 'warning', 'critical'): - ret['message'] = 'State must be any, unknown, passing, warning, or critical.' - ret['res'] = False + if state not in ("any", "unknown", "passing", "warning", "critical"): + ret["message"] = "State must be any, unknown, passing, warning, or critical." + ret["res"] = False return ret - function = 'health/state/{0}'.format(state) - ret = _query(consul_url=consul_url, - function=function, - token=token, - query_params=query_params) + function = "health/state/{0}".format(state) + ret = _query( + consul_url=consul_url, function=function, token=token, query_params=query_params + ) return ret def status_leader(consul_url=None, token=None): - ''' + """ Returns the current Raft leader :param consul_url: The Consul server URL. @@ -2013,25 +2008,23 @@ def status_leader(consul_url=None, token=None): salt '*' consul.status_leader - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'status/leader' - ret = _query(consul_url=consul_url, - function=function, - token=token) + function = "status/leader" + ret = _query(consul_url=consul_url, function=function, token=token) return ret def status_peers(consul_url, token=None): - ''' + """ Returns the current Raft peer set :param consul_url: The Consul server URL. @@ -2044,25 +2037,23 @@ def status_peers(consul_url, token=None): salt '*' consul.status_peers - ''' + """ ret = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - function = 'status/peers' - ret = _query(consul_url=consul_url, - function=function, - token=token) + function = "status/peers" + ret = _query(consul_url=consul_url, function=function, token=token) return ret def acl_create(consul_url=None, token=None, **kwargs): - ''' + """ Create a new ACL token. :param consul_url: The Consul server URL. @@ -2080,47 +2071,44 @@ def acl_create(consul_url=None, token=None, **kwargs): salt '*' consul.acl_create - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'name' in kwargs: - data['Name'] = kwargs['name'] + if "name" in kwargs: + data["Name"] = kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - if 'type' in kwargs: - data['Type'] = kwargs['type'] + if "type" in kwargs: + data["Type"] = kwargs["type"] - if 'rules' in kwargs: - data['Rules'] = kwargs['rules'] + if "rules" in kwargs: + data["Rules"] = kwargs["rules"] - function = 'acl/create' - res = _query(consul_url=consul_url, - token=token, - data=data, - method='PUT', - function=function) + function = "acl/create" + res = _query( + consul_url=consul_url, token=token, data=data, method="PUT", function=function + ) - if res['res']: - ret['res'] = True - ret['message'] = 'ACL {0} created.'.format(kwargs['name']) + if res["res"]: + ret["res"] = True + ret["message"] = "ACL {0} created.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = ('Removing Catalog ' - 'item {0} failed.'.format(kwargs['name'])) + ret["res"] = False + ret["message"] = "Removing Catalog " "item {0} failed.".format(kwargs["name"]) return ret def acl_update(consul_url=None, token=None, **kwargs): - ''' + """ Update an ACL token. :param consul_url: The Consul server URL. @@ -2139,55 +2127,52 @@ def acl_update(consul_url=None, token=None, **kwargs): salt '*' consul.acl_update - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'id' in kwargs: - data['ID'] = kwargs['id'] + if "id" in kwargs: + data["ID"] = kwargs["id"] else: - ret['message'] = 'Required parameter "id" is missing.' - ret['res'] = False + ret["message"] = 'Required parameter "id" is missing.' + ret["res"] = False return ret - if 'name' in kwargs: - data['Name'] = kwargs['name'] + if "name" in kwargs: + data["Name"] = kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - if 'type' in kwargs: - data['Type'] = kwargs['type'] + if "type" in kwargs: + data["Type"] = kwargs["type"] - if 'rules' in kwargs: - data['Rules'] = kwargs['rules'] + if "rules" in kwargs: + data["Rules"] = kwargs["rules"] - function = 'acl/update' - res = _query(consul_url=consul_url, - token=token, - data=data, - method='PUT', - function=function) + function = "acl/update" + res = _query( + consul_url=consul_url, token=token, data=data, method="PUT", function=function + ) - if res['res']: - ret['res'] = True - ret['message'] = 'ACL {0} created.'.format(kwargs['name']) + if res["res"]: + ret["res"] = True + ret["message"] = "ACL {0} created.".format(kwargs["name"]) else: - ret['res'] = False - ret['message'] = ('Adding ACL ' - '{0} failed.'.format(kwargs['name'])) + ret["res"] = False + ret["message"] = "Adding ACL " "{0} failed.".format(kwargs["name"]) return ret def acl_delete(consul_url=None, token=None, **kwargs): - ''' + """ Delete an ACL token. :param consul_url: The Consul server URL. @@ -2200,42 +2185,39 @@ def acl_delete(consul_url=None, token=None, **kwargs): salt '*' consul.acl_delete id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'id' not in kwargs: - ret['message'] = 'Required parameter "id" is missing.' - ret['res'] = False + if "id" not in kwargs: + ret["message"] = 'Required parameter "id" is missing.' + ret["res"] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) - res = _query(consul_url=consul_url, - token=token, - data=data, - method='PUT', - function=function) + function = "acl/delete/{0}".format(kwargs["id"]) + res = _query( + consul_url=consul_url, token=token, data=data, method="PUT", function=function + ) - if res['res']: - ret['res'] = True - ret['message'] = 'ACL {0} deleted.'.format(kwargs['id']) + if res["res"]: + ret["res"] = True + ret["message"] = "ACL {0} deleted.".format(kwargs["id"]) else: - ret['res'] = False - ret['message'] = ('Removing ACL ' - '{0} failed.'.format(kwargs['id'])) + ret["res"] = False + ret["message"] = "Removing ACL " "{0} failed.".format(kwargs["id"]) return ret def acl_info(consul_url=None, **kwargs): - ''' + """ Information about an ACL token. :param consul_url: The Consul server URL. @@ -2248,32 +2230,29 @@ def acl_info(consul_url=None, **kwargs): salt '*' consul.acl_info id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'id' not in kwargs: - ret['message'] = 'Required parameter "id" is missing.' - ret['res'] = False + if "id" not in kwargs: + ret["message"] = 'Required parameter "id" is missing.' + ret["res"] = False return ret - function = 'acl/info/{0}'.format(kwargs['id']) - ret = _query(consul_url=consul_url, - data=data, - method='GET', - function=function) + function = "acl/info/{0}".format(kwargs["id"]) + ret = _query(consul_url=consul_url, data=data, method="GET", function=function) return ret def acl_clone(consul_url=None, token=None, **kwargs): - ''' + """ Information about an ACL token. :param consul_url: The Consul server URL. @@ -2287,41 +2266,38 @@ def acl_clone(consul_url=None, token=None, **kwargs): salt '*' consul.acl_info id='c1c4d223-91cb-3d1f-1ee8-f2af9e7b6716' - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'id' not in kwargs: - ret['message'] = 'Required parameter "id" is missing.' - ret['res'] = False + if "id" not in kwargs: + ret["message"] = 'Required parameter "id" is missing.' + ret["res"] = False return ret - function = 'acl/clone/{0}'.format(kwargs['id']) - res = _query(consul_url=consul_url, - token=token, - data=data, - method='PUT', - function=function) - if res['res']: - ret['res'] = True - ret['message'] = 'ACL {0} cloned.'.format(kwargs['name']) - ret['ID'] = ret['data'] + function = "acl/clone/{0}".format(kwargs["id"]) + res = _query( + consul_url=consul_url, token=token, data=data, method="PUT", function=function + ) + if res["res"]: + ret["res"] = True + ret["message"] = "ACL {0} cloned.".format(kwargs["name"]) + ret["ID"] = ret["data"] else: - ret['res'] = False - ret['message'] = ('Cloning ACL' - 'item {0} failed.'.format(kwargs['name'])) + ret["res"] = False + ret["message"] = "Cloning ACL" "item {0} failed.".format(kwargs["name"]) return ret def acl_list(consul_url=None, token=None, **kwargs): - ''' + """ List the ACL tokens. :param consul_url: The Consul server URL. @@ -2333,33 +2309,31 @@ def acl_list(consul_url=None, token=None, **kwargs): salt '*' consul.acl_list - ''' + """ ret = {} data = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'id' not in kwargs: - ret['message'] = 'Required parameter "id" is missing.' - ret['res'] = False + if "id" not in kwargs: + ret["message"] = 'Required parameter "id" is missing.' + ret["res"] = False return ret - function = 'acl/list' - ret = _query(consul_url=consul_url, - token=token, - data=data, - method='PUT', - function=function) + function = "acl/list" + ret = _query( + consul_url=consul_url, token=token, data=data, method="PUT", function=function + ) return ret def event_fire(consul_url=None, token=None, name=None, **kwargs): - ''' + """ List the ACL tokens. :param consul_url: The Consul server URL. @@ -2377,52 +2351,53 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): salt '*' consul.event_fire name='deploy' - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret if not name: raise SaltInvocationError('Required argument "name" is missing.') - if 'dc' in kwargs: - query_params = kwargs['dc'] + if "dc" in kwargs: + query_params = kwargs["dc"] - if 'node' in kwargs: - query_params = kwargs['node'] + if "node" in kwargs: + query_params = kwargs["node"] - if 'service' in kwargs: - query_params = kwargs['service'] + if "service" in kwargs: + query_params = kwargs["service"] - if 'tag' in kwargs: - query_params = kwargs['tag'] + if "tag" in kwargs: + query_params = kwargs["tag"] - function = 'event/fire/{0}'.format(name) - res = _query(consul_url=consul_url, - token=token, - query_params=query_params, - method='PUT', - function=function) + function = "event/fire/{0}".format(name) + res = _query( + consul_url=consul_url, + token=token, + query_params=query_params, + method="PUT", + function=function, + ) - if res['res']: - ret['res'] = True - ret['message'] = 'Event {0} fired.'.format(name) - ret['data'] = ret['data'] + if res["res"]: + ret["res"] = True + ret["message"] = "Event {0} fired.".format(name) + ret["data"] = ret["data"] else: - ret['res'] = False - ret['message'] = ('Cloning ACL' - 'item {0} failed.'.format(kwargs['name'])) + ret["res"] = False + ret["message"] = "Cloning ACL" "item {0} failed.".format(kwargs["name"]) return ret def event_list(consul_url=None, token=None, **kwargs): - ''' + """ List the recent events. :param consul_url: The Consul server URL. @@ -2435,25 +2410,24 @@ def event_list(consul_url=None, token=None, **kwargs): salt '*' consul.event_list - ''' + """ ret = {} query_params = {} if not consul_url: consul_url = _get_config() if not consul_url: - log.error('No Consul URL found.') - ret['message'] = 'No Consul URL found.' - ret['res'] = False + log.error("No Consul URL found.") + ret["message"] = "No Consul URL found." + ret["res"] = False return ret - if 'name' in kwargs: - query_params = kwargs['name'] + if "name" in kwargs: + query_params = kwargs["name"] else: raise SaltInvocationError('Required argument "name" is missing.') - function = 'event/list/' - ret = _query(consul_url=consul_url, - token=token, - query_params=query_params, - function=function) + function = "event/list/" + ret = _query( + consul_url=consul_url, token=token, query_params=query_params, function=function + ) return ret diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 6b36cda..c411b66 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1,23 +1,25 @@ # -*- coding: utf-8 -*- -''' +""" Render the pillar data -''' +""" # Import python libs from __future__ import absolute_import, print_function, unicode_literals + +import collections import copy import fnmatch -import os -import collections +import inspect import logging -import salt.ext.tornado.gen +import os import sys import traceback -import inspect + +import salt.ext.tornado.gen +import salt.fileclient # Import salt libs import salt.loader -import salt.fileclient import salt.minion import salt.transport.client import salt.utils.args @@ -27,81 +29,121 @@ import salt.utils.dictupdate import salt.utils.url from salt.exceptions import SaltClientError + +# Import 3rd-party libs +from salt.ext import six from salt.template import compile_template -from salt.utils.odict import OrderedDict -from salt.version import __version__ + # Even though dictupdate is imported, invoking salt.utils.dictupdate.merge here # causes an UnboundLocalError. This should be investigated and fixed, but until # then, leave the import directly below this comment intact. from salt.utils.dictupdate import merge - -# Import 3rd-party libs -from salt.ext import six +from salt.utils.odict import OrderedDict +from salt.version import __version__ log = logging.getLogger(__name__) -def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None, extra_minion_data=None): - ''' +def get_pillar( + opts, + grains, + minion_id, + saltenv=None, + ext=None, + funcs=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, +): + """ Return the correct pillar driver based on the file_client option - ''' - file_client = opts['file_client'] - if opts.get('master_type') == 'disable' and file_client == 'remote': - file_client = 'local' - ptype = { - 'remote': RemotePillar, - 'local': Pillar - }.get(file_client, Pillar) + """ + file_client = opts["file_client"] + if opts.get("master_type") == "disable" and file_client == "remote": + file_client = "local" + ptype = {"remote": RemotePillar, "local": Pillar}.get(file_client, Pillar) # If local pillar and we're caching, run through the cache system first - log.debug('Determining pillar cache') - if opts['pillar_cache']: - log.info('Compiling pillar from cache') - log.debug('get_pillar using pillar cache with ext: %s', ext) - return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv) - return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv, - extra_minion_data=extra_minion_data) + log.debug("Determining pillar cache") + if opts["pillar_cache"]: + log.info("Compiling pillar from cache") + log.debug("get_pillar using pillar cache with ext: %s", ext) + return PillarCache( + opts, + grains, + minion_id, + saltenv, + ext=ext, + functions=funcs, + pillar_override=pillar_override, + pillarenv=pillarenv, + ) + return ptype( + opts, + grains, + minion_id, + saltenv, + ext, + functions=funcs, + pillar_override=pillar_override, + pillarenv=pillarenv, + extra_minion_data=extra_minion_data, + ) # TODO: migrate everyone to this one! -def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar_override=None, pillarenv=None, - extra_minion_data=None): - ''' +def get_async_pillar( + opts, + grains, + minion_id, + saltenv=None, + ext=None, + funcs=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, +): + """ Return the correct pillar driver based on the file_client option - ''' - file_client = opts['file_client'] - if opts.get('master_type') == 'disable' and file_client == 'remote': - file_client = 'local' - ptype = { - 'remote': AsyncRemotePillar, - 'local': AsyncPillar, - }.get(file_client, AsyncPillar) - return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar_override=pillar_override, pillarenv=pillarenv, - extra_minion_data=extra_minion_data) + """ + file_client = opts["file_client"] + if opts.get("master_type") == "disable" and file_client == "remote": + file_client = "local" + ptype = {"remote": AsyncRemotePillar, "local": AsyncPillar}.get( + file_client, AsyncPillar + ) + return ptype( + opts, + grains, + minion_id, + saltenv, + ext, + functions=funcs, + pillar_override=pillar_override, + pillarenv=pillarenv, + extra_minion_data=extra_minion_data, + ) class RemotePillarMixin(object): - ''' + """ Common remote pillar functionality - ''' + """ + def get_ext_pillar_extra_minion_data(self, opts): - ''' + """ Returns the extra data from the minion's opts dict (the config file). This data will be passed to external pillar functions. - ''' + """ + def get_subconfig(opts_key): - ''' + """ Returns a dict containing the opts key subtree, while maintaining the opts structure - ''' + """ ret_dict = aux_dict = {} config_val = opts - subkeys = opts_key.split(':') + subkeys = opts_key.split(":") # Build an empty dict with the opts path for subkey in subkeys[:-1]: aux_dict[subkey] = {} @@ -116,75 +158,91 @@ def get_subconfig(opts_key): return ret_dict extra_data = {} - if 'pass_to_ext_pillars' in opts: - if not isinstance(opts['pass_to_ext_pillars'], list): - log.exception('\'pass_to_ext_pillars\' config is malformed.') - raise SaltClientError('\'pass_to_ext_pillars\' config is ' - 'malformed.') - for key in opts['pass_to_ext_pillars']: - salt.utils.dictupdate.update(extra_data, - get_subconfig(key), - recursive_update=True, - merge_lists=True) - log.trace('ext_pillar_extra_data = %s', extra_data) + if "pass_to_ext_pillars" in opts: + if not isinstance(opts["pass_to_ext_pillars"], list): + log.exception("'pass_to_ext_pillars' config is malformed.") + raise SaltClientError("'pass_to_ext_pillars' config is " "malformed.") + for key in opts["pass_to_ext_pillars"]: + salt.utils.dictupdate.update( + extra_data, + get_subconfig(key), + recursive_update=True, + merge_lists=True, + ) + log.trace("ext_pillar_extra_data = %s", extra_data) return extra_data class AsyncRemotePillar(RemotePillarMixin): - ''' + """ Get the pillar from the master - ''' - def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None, extra_minion_data=None): + """ + + def __init__( + self, + opts, + grains, + minion_id, + saltenv, + ext=None, + functions=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, + ): self.opts = opts - self.opts['saltenv'] = saltenv + self.opts["saltenv"] = saltenv self.ext = ext self.grains = grains self.minion_id = minion_id self.channel = salt.transport.client.AsyncReqChannel.factory(opts) if pillarenv is not None: - self.opts['pillarenv'] = pillarenv + self.opts["pillarenv"] = pillarenv self.pillar_override = pillar_override or {} if not isinstance(self.pillar_override, dict): self.pillar_override = {} - log.error('Pillar data must be a dictionary') + log.error("Pillar data must be a dictionary") self.extra_minion_data = extra_minion_data or {} if not isinstance(self.extra_minion_data, dict): self.extra_minion_data = {} - log.error('Extra minion data must be a dictionary') - salt.utils.dictupdate.update(self.extra_minion_data, - self.get_ext_pillar_extra_minion_data(opts), - recursive_update=True, - merge_lists=True) + log.error("Extra minion data must be a dictionary") + salt.utils.dictupdate.update( + self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True, + ) self._closing = False @salt.ext.tornado.gen.coroutine def compile_pillar(self): - ''' + """ Return a future which will contain the pillar data from the master - ''' - load = {'id': self.minion_id, - 'grains': self.grains, - 'saltenv': self.opts['saltenv'], - 'pillarenv': self.opts['pillarenv'], - 'pillar_override': self.pillar_override, - 'extra_minion_data': self.extra_minion_data, - 'ver': '2', - 'cmd': '_pillar'} + """ + load = { + "id": self.minion_id, + "grains": self.grains, + "saltenv": self.opts["saltenv"], + "pillarenv": self.opts["pillarenv"], + "pillar_override": self.pillar_override, + "extra_minion_data": self.extra_minion_data, + "ver": "2", + "cmd": "_pillar", + } if self.ext: - load['ext'] = self.ext + load["ext"] = self.ext try: ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( - load, - dictkey='pillar', + load, dictkey="pillar", ) except Exception: # pylint: disable=broad-except - log.exception('Exception getting pillar:') - raise SaltClientError('Exception getting pillar.') + log.exception("Exception getting pillar:") + raise SaltClientError("Exception getting pillar.") if not isinstance(ret_pillar, dict): - msg = ('Got a bad pillar from master, type {0}, expecting dict: ' - '{1}').format(type(ret_pillar).__name__, ret_pillar) + msg = ( + "Got a bad pillar from master, type {0}, expecting dict: " "{1}" + ).format(type(ret_pillar).__name__, ret_pillar) log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! raise SaltClientError(msg) @@ -200,65 +258,82 @@ def destroy(self): # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class RemotePillar(RemotePillarMixin): - ''' + """ Get the pillar from the master - ''' - def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None, extra_minion_data=None): + """ + + def __init__( + self, + opts, + grains, + minion_id, + saltenv, + ext=None, + functions=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, + ): self.opts = opts - self.opts['saltenv'] = saltenv + self.opts["saltenv"] = saltenv self.ext = ext self.grains = grains self.minion_id = minion_id self.channel = salt.transport.client.ReqChannel.factory(opts) if pillarenv is not None: - self.opts['pillarenv'] = pillarenv + self.opts["pillarenv"] = pillarenv self.pillar_override = pillar_override or {} if not isinstance(self.pillar_override, dict): self.pillar_override = {} - log.error('Pillar data must be a dictionary') + log.error("Pillar data must be a dictionary") self.extra_minion_data = extra_minion_data or {} if not isinstance(self.extra_minion_data, dict): self.extra_minion_data = {} - log.error('Extra minion data must be a dictionary') - salt.utils.dictupdate.update(self.extra_minion_data, - self.get_ext_pillar_extra_minion_data(opts), - recursive_update=True, - merge_lists=True) + log.error("Extra minion data must be a dictionary") + salt.utils.dictupdate.update( + self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True, + ) self._closing = False def compile_pillar(self): - ''' + """ Return the pillar data from the master - ''' - load = {'id': self.minion_id, - 'grains': self.grains, - 'saltenv': self.opts['saltenv'], - 'pillarenv': self.opts['pillarenv'], - 'pillar_override': self.pillar_override, - 'extra_minion_data': self.extra_minion_data, - 'ver': '2', - 'cmd': '_pillar'} + """ + load = { + "id": self.minion_id, + "grains": self.grains, + "saltenv": self.opts["saltenv"], + "pillarenv": self.opts["pillarenv"], + "pillar_override": self.pillar_override, + "extra_minion_data": self.extra_minion_data, + "ver": "2", + "cmd": "_pillar", + } if self.ext: - load['ext'] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry(load, - dictkey='pillar', - ) + load["ext"] = self.ext + ret_pillar = self.channel.crypted_transfer_decode_dictentry( + load, dictkey="pillar", + ) if not isinstance(ret_pillar, dict): log.error( - 'Got a bad pillar from master, type %s, expecting dict: %s', - type(ret_pillar).__name__, ret_pillar + "Got a bad pillar from master, type %s, expecting dict: %s", + type(ret_pillar).__name__, + ret_pillar, ) return {} return ret_pillar def destroy(self): - if hasattr(self, '_closing') and self._closing: + if hasattr(self, "_closing") and self._closing: return self._closing = True @@ -267,11 +342,12 @@ def destroy(self): # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 class PillarCache(object): - ''' + """ Return a cached pillar if it exists, otherwise cache it. Pillar caches are structed in two diminensions: minion_id with a dict of @@ -283,10 +359,21 @@ class PillarCache(object): {'minion_1': {'base': {'pilar_key_1' 'pillar_val_1'} } - ''' + """ + # TODO ABC? - def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None, extra_minion_data=None): + def __init__( + self, + opts, + grains, + minion_id, + saltenv, + ext=None, + functions=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, + ): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -300,214 +387,240 @@ def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, self.pillarenv = pillarenv if saltenv is None: - self.saltenv = 'base' + self.saltenv = "base" else: self.saltenv = saltenv # Determine caching backend self.cache = salt.utils.cache.CacheFactory.factory( - self.opts['pillar_cache_backend'], - self.opts['pillar_cache_ttl'], - minion_cache_path=self._minion_cache_path(minion_id)) + self.opts["pillar_cache_backend"], + self.opts["pillar_cache_ttl"], + minion_cache_path=self._minion_cache_path(minion_id), + ) def _minion_cache_path(self, minion_id): - ''' + """ Return the path to the cache file for the minion. Used only for disk-based backends - ''' - return os.path.join(self.opts['cachedir'], 'pillar_cache', minion_id) + """ + return os.path.join(self.opts["cachedir"], "pillar_cache", minion_id) def fetch_pillar(self): - ''' + """ In the event of a cache miss, we need to incur the overhead of caching a new pillar. - ''' - log.debug('Pillar cache getting external pillar with ext: %s', self.ext) - fresh_pillar = Pillar(self.opts, - self.grains, - self.minion_id, - self.saltenv, - ext=self.ext, - functions=self.functions, - pillar_override=self.pillar_override, - pillarenv=self.pillarenv) + """ + log.debug("Pillar cache getting external pillar with ext: %s", self.ext) + fresh_pillar = Pillar( + self.opts, + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar_override=self.pillar_override, + pillarenv=self.pillarenv, + ) return fresh_pillar.compile_pillar() def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs - log.debug('Scanning pillar cache for information about minion %s and pillarenv %s', self.minion_id, self.pillarenv) - log.debug('Scanning cache: %s', self.cache._dict) + log.debug( + "Scanning pillar cache for information about minion %s and pillarenv %s", + self.minion_id, + self.pillarenv, + ) + log.debug("Scanning cache: %s", self.cache._dict) # Check the cache! if self.minion_id in self.cache: # Keyed by minion_id # TODO Compare grains, etc? if self.pillarenv in self.cache[self.minion_id]: # We have a cache hit! Send it back. - log.debug('Pillar cache hit for minion %s and pillarenv %s', self.minion_id, self.pillarenv) + log.debug( + "Pillar cache hit for minion %s and pillarenv %s", + self.minion_id, + self.pillarenv, + ) return self.cache[self.minion_id][self.pillarenv] else: # We found the minion but not the env. Store it. fresh_pillar = self.fetch_pillar() self.cache[self.minion_id][self.pillarenv] = fresh_pillar - log.debug('Pillar cache miss for pillarenv %s for minion %s', self.pillarenv, self.minion_id) + log.debug( + "Pillar cache miss for pillarenv %s for minion %s", + self.pillarenv, + self.minion_id, + ) return fresh_pillar else: # We haven't seen this minion yet in the cache. Store it. fresh_pillar = self.fetch_pillar() self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} - log.debug('Pillar cache miss for minion %s', self.minion_id) - log.debug('Current pillar cache: %s', self.cache._dict) # FIXME hack! + log.debug("Pillar cache miss for minion %s", self.minion_id) + log.debug("Current pillar cache: %s", self.cache._dict) # FIXME hack! return fresh_pillar class Pillar(object): - ''' + """ Read over the pillar top files and render the pillar data - ''' - def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar_override=None, pillarenv=None, extra_minion_data=None): + """ + + def __init__( + self, + opts, + grains, + minion_id, + saltenv, + ext=None, + functions=None, + pillar_override=None, + pillarenv=None, + extra_minion_data=None, + ): self.minion_id = minion_id self.ext = ext if pillarenv is None: - if opts.get('pillarenv_from_saltenv', False): - opts['pillarenv'] = saltenv + if opts.get("pillarenv_from_saltenv", False): + opts["pillarenv"] = saltenv # use the local file client self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv) self.saltenv = saltenv self.client = salt.fileclient.get_file_client(self.opts, True) self.avail = self.__gather_avail() - if opts.get('file_client', '') == 'local': - opts['grains'] = grains + if opts.get("file_client", "") == "local": + opts["grains"] = grains # if we didn't pass in functions, lets load them if functions is None: utils = salt.loader.utils(opts) - if opts.get('file_client', '') == 'local': + if opts.get("file_client", "") == "local": self.functions = salt.loader.minion_mods(opts, utils=utils) else: self.functions = salt.loader.minion_mods(self.opts, utils=utils) else: self.functions = functions - self.opts['minion_id'] = minion_id + self.opts["minion_id"] = minion_id self.matchers = salt.loader.matchers(self.opts) self.rend = salt.loader.render(self.opts, self.functions) ext_pillar_opts = copy.deepcopy(self.opts) # Keep the incoming opts ID intact, ie, the master id - if 'id' in opts: - ext_pillar_opts['id'] = opts['id'] - self.merge_strategy = 'smart' - if opts.get('pillar_source_merging_strategy'): - self.merge_strategy = opts['pillar_source_merging_strategy'] + if "id" in opts: + ext_pillar_opts["id"] = opts["id"] + self.merge_strategy = "smart" + if opts.get("pillar_source_merging_strategy"): + self.merge_strategy = opts["pillar_source_merging_strategy"] self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} self.pillar_override = pillar_override or {} if not isinstance(self.pillar_override, dict): self.pillar_override = {} - log.error('Pillar data must be a dictionary') + log.error("Pillar data must be a dictionary") self.extra_minion_data = extra_minion_data or {} if not isinstance(self.extra_minion_data, dict): self.extra_minion_data = {} - log.error('Extra minion data must be a dictionary') + log.error("Extra minion data must be a dictionary") self._closing = False def __valid_on_demand_ext_pillar(self, opts): - ''' + """ Check to see if the on demand external pillar is allowed - ''' + """ if not isinstance(self.ext, dict): - log.error( - 'On-demand pillar %s is not formatted as a dictionary', - self.ext - ) + log.error("On-demand pillar %s is not formatted as a dictionary", self.ext) return False - on_demand = opts.get('on_demand_ext_pillar', []) + on_demand = opts.get("on_demand_ext_pillar", []) try: invalid_on_demand = set([x for x in self.ext if x not in on_demand]) except TypeError: # Prevent traceback when on_demand_ext_pillar option is malformed log.error( - 'The \'on_demand_ext_pillar\' configuration option is ' - 'malformed, it should be a list of ext_pillar module names' + "The 'on_demand_ext_pillar' configuration option is " + "malformed, it should be a list of ext_pillar module names" ) return False if invalid_on_demand: log.error( - 'The following ext_pillar modules are not allowed for ' - 'on-demand pillar data: %s. Valid on-demand ext_pillar ' - 'modules are: %s. The valid modules can be adjusted by ' - 'setting the \'on_demand_ext_pillar\' config option.', - ', '.join(sorted(invalid_on_demand)), - ', '.join(on_demand), + "The following ext_pillar modules are not allowed for " + "on-demand pillar data: %s. Valid on-demand ext_pillar " + "modules are: %s. The valid modules can be adjusted by " + "setting the 'on_demand_ext_pillar' config option.", + ", ".join(sorted(invalid_on_demand)), + ", ".join(on_demand), ) return False return True def __gather_avail(self): - ''' + """ Gather the lists of available sls data from the master - ''' + """ avail = {} for saltenv in self._get_envs(): avail[saltenv] = self.client.list_states(saltenv) return avail def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): - ''' + """ The options need to be altered to conform to the file client - ''' + """ opts = copy.deepcopy(opts_in) - opts['file_client'] = 'local' + opts["file_client"] = "local" if not grains: - opts['grains'] = {} + opts["grains"] = {} else: - opts['grains'] = grains + opts["grains"] = grains # Allow minion/CLI saltenv/pillarenv to take precedence over master - opts['saltenv'] = saltenv \ - if saltenv is not None \ - else opts.get('saltenv') - opts['pillarenv'] = pillarenv \ - if pillarenv is not None \ - else opts.get('pillarenv') - opts['id'] = self.minion_id - if opts['state_top'].startswith('salt://'): - opts['state_top'] = opts['state_top'] - elif opts['state_top'].startswith('/'): - opts['state_top'] = salt.utils.url.create(opts['state_top'][1:]) + opts["saltenv"] = saltenv if saltenv is not None else opts.get("saltenv") + opts["pillarenv"] = ( + pillarenv if pillarenv is not None else opts.get("pillarenv") + ) + opts["id"] = self.minion_id + if opts["state_top"].startswith("salt://"): + opts["state_top"] = opts["state_top"] + elif opts["state_top"].startswith("/"): + opts["state_top"] = salt.utils.url.create(opts["state_top"][1:]) else: - opts['state_top'] = salt.utils.url.create(opts['state_top']) + opts["state_top"] = salt.utils.url.create(opts["state_top"]) if self.ext and self.__valid_on_demand_ext_pillar(opts): - if 'ext_pillar' in opts: - opts['ext_pillar'].append(self.ext) + if "ext_pillar" in opts: + opts["ext_pillar"].append(self.ext) else: - opts['ext_pillar'] = [self.ext] - if '__env__' in opts['pillar_roots']: - env = opts.get('pillarenv') or opts.get('saltenv') or 'base' - if env not in opts['pillar_roots']: - log.debug("pillar environment '%s' maps to __env__ pillar_roots directory", env) - opts['pillar_roots'][env] = opts['pillar_roots'].pop('__env__') + opts["ext_pillar"] = [self.ext] + if "__env__" in opts["pillar_roots"]: + env = opts.get("pillarenv") or opts.get("saltenv") or "base" + if env not in opts["pillar_roots"]: + log.debug( + "pillar environment '%s' maps to __env__ pillar_roots directory", + env, + ) + opts["pillar_roots"][env] = opts["pillar_roots"].pop("__env__") else: - log.debug("pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", - env) - opts['pillar_roots'].pop('__env__') + log.debug( + "pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", + env, + ) + opts["pillar_roots"].pop("__env__") return opts def _get_envs(self): - ''' + """ Pull the file server environments out of the master options - ''' - envs = set(['base']) - if 'pillar_roots' in self.opts: - envs.update(list(self.opts['pillar_roots'])) + """ + envs = set(["base"]) + if "pillar_roots" in self.opts: + envs.update(list(self.opts["pillar_roots"])) return envs def get_tops(self): - ''' + """ Gather the top files - ''' + """ tops = collections.defaultdict(list) include = collections.defaultdict(list) done = collections.defaultdict(list) @@ -515,48 +628,51 @@ def get_tops(self): # Gather initial top files try: saltenvs = set() - if self.opts['pillarenv']: + if self.opts["pillarenv"]: # If the specified pillarenv is not present in the available # pillar environments, do not cache the pillar top file. - if self.opts['pillarenv'] not in self.opts['pillar_roots']: + if self.opts["pillarenv"] not in self.opts["pillar_roots"]: log.debug( - 'pillarenv \'%s\' not found in the configured pillar ' - 'environments (%s)', - self.opts['pillarenv'], ', '.join(self.opts['pillar_roots']) + "pillarenv '%s' not found in the configured pillar " + "environments (%s)", + self.opts["pillarenv"], + ", ".join(self.opts["pillar_roots"]), ) else: - saltenvs.add(self.opts['pillarenv']) + saltenvs.add(self.opts["pillarenv"]) else: saltenvs = self._get_envs() - if self.opts.get('pillar_source_merging_strategy', None) == "none": - saltenvs &= set([self.saltenv or 'base']) + if self.opts.get("pillar_source_merging_strategy", None) == "none": + saltenvs &= set([self.saltenv or "base"]) for saltenv in saltenvs: - top = self.client.cache_file(self.opts['state_top'], saltenv) + top = self.client.cache_file(self.opts["state_top"], saltenv) if top: - tops[saltenv].append(compile_template( - top, - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - saltenv=saltenv, - _pillar_rend=True, - )) + tops[saltenv].append( + compile_template( + top, + self.rend, + self.opts["renderer"], + self.opts["renderer_blacklist"], + self.opts["renderer_whitelist"], + saltenv=saltenv, + _pillar_rend=True, + ) + ) except Exception as exc: # pylint: disable=broad-except errors.append( - ('Rendering Primary Top file failed, render error:\n{0}' - .format(exc))) - log.exception('Pillar rendering failed for minion %s', self.minion_id) + ("Rendering Primary Top file failed, render error:\n{0}".format(exc)) + ) + log.exception("Pillar rendering failed for minion %s", self.minion_id) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): for ctop in ctops: - if 'include' not in ctop: + if "include" not in ctop: continue - for sls in ctop['include']: + for sls in ctop["include"]: include[saltenv].append(sls) - ctop.pop('include') + ctop.pop("include") # Go through the includes and pull out the extra tops and add them while include: pops = [] @@ -569,23 +685,22 @@ def get_tops(self): continue try: tops[saltenv].append( - compile_template( - self.client.get_state( - sls, - saltenv - ).get('dest', False), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - saltenv=saltenv, - _pillar_rend=True, - ) - ) + compile_template( + self.client.get_state(sls, saltenv).get("dest", False), + self.rend, + self.opts["renderer"], + self.opts["renderer_blacklist"], + self.opts["renderer_whitelist"], + saltenv=saltenv, + _pillar_rend=True, + ) + ) except Exception as exc: # pylint: disable=broad-except errors.append( - ('Rendering Top file {0} failed, render error' - ':\n{1}').format(sls, exc)) + ( + "Rendering Top file {0} failed, render error" ":\n{1}" + ).format(sls, exc) + ) done[saltenv].append(sls) for saltenv in pops: if saltenv in include: @@ -594,15 +709,15 @@ def get_tops(self): return tops, errors def merge_tops(self, tops): - ''' + """ Cleanly merge the top files - ''' + """ top = collections.defaultdict(OrderedDict) orders = collections.defaultdict(OrderedDict) for ctops in six.itervalues(tops): for ctop in ctops: for saltenv, targets in six.iteritems(ctop): - if saltenv == 'include': + if saltenv == "include": continue for tgt in targets: matches = [] @@ -611,17 +726,17 @@ def merge_tops(self, tops): ignore_missing = False for comp in ctop[saltenv][tgt]: if isinstance(comp, dict): - if 'match' in comp: + if "match" in comp: matches.append(comp) - if 'order' in comp: - order = comp['order'] + if "order" in comp: + order = comp["order"] if not isinstance(order, int): try: order = int(order) except ValueError: order = 0 orders[saltenv][tgt] = order - if comp.get('ignore_missing', False): + if comp.get("ignore_missing", False): ignore_missing = True if isinstance(comp, six.string_types): states[comp] = True @@ -634,33 +749,32 @@ def merge_tops(self, tops): return self.sort_top_targets(top, orders) def sort_top_targets(self, top, orders): - ''' + """ Returns the sorted high data from the merged top files - ''' + """ sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop for saltenv, targets in six.iteritems(top): - sorted_targets = sorted(targets, - key=lambda target: orders[saltenv][target]) + sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: sorted_top[saltenv][target] = targets[target] # pylint: enable=cell-var-from-loop return sorted_top def get_top(self): - ''' + """ Returns the high data derived from the top file - ''' + """ tops, errors = self.get_tops() try: merged_tops = self.merge_tops(tops) except TypeError as err: merged_tops = OrderedDict() - errors.append('Error encountered while rendering pillar top file.') + errors.append("Error encountered while rendering pillar top file.") return merged_tops, errors def top_matches(self, top, reload=False): - ''' + """ Search through the top high data for matches and return the states that this minion needs to execute. @@ -669,89 +783,97 @@ def top_matches(self, top, reload=False): reload Reload the matcher loader - ''' + """ matches = {} if reload: self.matchers = salt.loader.matchers(self.opts) for saltenv, body in six.iteritems(top): - if self.opts['pillarenv']: - if saltenv != self.opts['pillarenv']: + if self.opts["pillarenv"]: + if saltenv != self.opts["pillarenv"]: continue for match, data in six.iteritems(body): - if self.matchers['confirm_top.confirm_top']( - match, - data, - self.opts.get('nodegroups', {}), - ): + if self.matchers["confirm_top.confirm_top"]( + match, data, self.opts.get("nodegroups", {}), + ): if saltenv not in matches: matches[saltenv] = env_matches = [] else: env_matches = matches[saltenv] for item in data: - if isinstance(item, six.string_types) and item not in env_matches: + if ( + isinstance(item, six.string_types) + and item not in env_matches + ): env_matches.append(item) return matches def render_pstate(self, sls, saltenv, mods, defaults=None): - ''' + """ Collect a single pillar sls file and render it - ''' + """ if defaults is None: defaults = {} - err = '' + err = "" errors = [] - fn_ = self.client.get_state(sls, saltenv).get('dest', False) + fn_ = self.client.get_state(sls, saltenv).get("dest", False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): - log.debug('Skipping ignored and missing SLS \'%s\' in ' - 'environment \'%s\'', sls, saltenv) + log.debug( + "Skipping ignored and missing SLS '%s' in " "environment '%s'", + sls, + saltenv, + ) return None, mods, errors - elif self.opts['pillar_roots'].get(saltenv): - msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not' - ' available on the salt master').format(sls, saltenv) + elif self.opts["pillar_roots"].get(saltenv): + msg = ( + "Specified SLS '{0}' in environment '{1}' is not" + " available on the salt master" + ).format(sls, saltenv) log.error(msg) errors.append(msg) else: - msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not ' - 'found. '.format(sls, saltenv)) - if self.opts.get('__git_pillar', False) is True: + msg = ( + "Specified SLS '{0}' in environment '{1}' was not " + "found. ".format(sls, saltenv) + ) + if self.opts.get("__git_pillar", False) is True: msg += ( - 'This is likely caused by a git_pillar top file ' - 'containing an environment other than the one for the ' - 'branch in which it resides. Each git_pillar ' - 'branch/tag must have its own top file.' + "This is likely caused by a git_pillar top file " + "containing an environment other than the one for the " + "branch in which it resides. Each git_pillar " + "branch/tag must have its own top file." ) else: msg += ( - 'This could be because SLS \'{0}\' is in an ' - 'environment other than \'{1}\', but \'{1}\' is ' - 'included in that environment\'s Pillar top file. It ' - 'could also be due to environment \'{1}\' not being ' - 'defined in \'pillar_roots\'.'.format(sls, saltenv) + "This could be because SLS '{0}' is in an " + "environment other than '{1}', but '{1}' is " + "included in that environment's Pillar top file. It " + "could also be due to environment '{1}' not being " + "defined in 'pillar_roots'.".format(sls, saltenv) ) log.debug(msg) # return state, mods, errors return None, mods, errors state = None try: - state = compile_template(fn_, - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - saltenv, - sls, - _pillar_rend=True, - **defaults) - except Exception as exc: # pylint: disable=broad-except - msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format( - sls, exc + state = compile_template( + fn_, + self.rend, + self.opts["renderer"], + self.opts["renderer_blacklist"], + self.opts["renderer_whitelist"], + saltenv, + sls, + _pillar_rend=True, + **defaults ) + except Exception as exc: # pylint: disable=broad-except + msg = "Rendering SLS '{0}' failed, render error:\n{1}".format(sls, exc) log.critical(msg, exc_info=True) - if self.opts.get('pillar_safe_render_error', True): + if self.opts.get("pillar_safe_render_error", True): errors.append( - 'Rendering SLS \'{0}\' failed. Please see master log for ' - 'details.'.format(sls) + "Rendering SLS '{0}' failed. Please see master log for " + "details.".format(sls) ) else: errors.append(msg) @@ -759,64 +881,70 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate = None if state: if not isinstance(state, dict): - msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls) + msg = "SLS '{0}' does not render to a dictionary".format(sls) log.error(msg) errors.append(msg) else: - if 'include' in state: - if not isinstance(state['include'], list): - msg = ('Include Declaration in SLS \'{0}\' is not ' - 'formed as a list'.format(sls)) + if "include" in state: + if not isinstance(state["include"], list): + msg = ( + "Include Declaration in SLS '{0}' is not " + "formed as a list".format(sls) + ) log.error(msg) errors.append(msg) else: # render included state(s) include_states = [] - for sub_sls in state.pop('include'): + for sub_sls in state.pop("include"): if isinstance(sub_sls, dict): sub_sls, v = next(six.iteritems(sub_sls)) - defaults = v.get('defaults', {}) - key = v.get('key', None) + defaults = v.get("defaults", {}) + key = v.get("key", None) else: key = None try: matched_pstates = fnmatch.filter( self.avail[saltenv], - sub_sls.lstrip('.').replace('/', '.'), + sub_sls.lstrip(".").replace("/", "."), ) except KeyError: errors.extend( - ['No matching pillar environment for environment ' - '\'{0}\' found'.format(saltenv)] + [ + "No matching pillar environment for environment " + "'{0}' found".format(saltenv) + ] ) matched_pstates = [sub_sls] for m_sub_sls in matched_pstates: if m_sub_sls not in mods: nstate, mods, err = self.render_pstate( - m_sub_sls, - saltenv, - mods, - defaults - ) + m_sub_sls, saltenv, mods, defaults + ) if nstate: if key: # If key is x:y, convert it to {x: {y: nstate}} - for key_fragment in reversed(key.split(":")): - nstate = { - key_fragment: nstate - } - if not self.opts.get('pillar_includes_override_sls', False): + for key_fragment in reversed( + key.split(":") + ): + nstate = {key_fragment: nstate} + if not self.opts.get( + "pillar_includes_override_sls", False + ): include_states.append(nstate) else: state = merge( state, nstate, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get( + "pillar_merge_lists", False + ), + ) if err: errors += err - if not self.opts.get('pillar_includes_override_sls', False): + if not self.opts.get("pillar_includes_override_sls", False): # merge included state(s) with the current state # merged last to ensure that its values are # authoritative. @@ -830,15 +958,16 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): state, s, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) return state, mods, errors def render_pillar(self, matches, errors=None): - ''' + """ Extract the sls pillar files from the matches and render them into the pillar - ''' + """ pillar = copy.copy(self.pillar_override) if errors is None: errors = [] @@ -851,8 +980,10 @@ def render_pillar(self, matches, errors=None): matched_pstates = fnmatch.filter(self.avail[saltenv], sls_match) except KeyError: errors.extend( - ['No matching pillar environment for environment ' - '\'{0}\' found'.format(saltenv)] + [ + "No matching pillar environment for environment " + "'{0}' found".format(saltenv) + ] ) if matched_pstates: pstatefiles.extend(matched_pstates) @@ -868,87 +999,92 @@ def render_pillar(self, matches, errors=None): if pstate is not None: if not isinstance(pstate, dict): log.error( - 'The rendered pillar sls file, \'%s\' state did ' - 'not return the expected data format. This is ' - 'a sign of a malformed pillar sls file. Returned ' - 'errors: %s', + "The rendered pillar sls file, '%s' state did " + "not return the expected data format. This is " + "a sign of a malformed pillar sls file. Returned " + "errors: %s", sls, - ', '.join(["'{0}'".format(e) for e in errors]) + ", ".join(["'{0}'".format(e) for e in errors]), ) continue pillar = merge( pillar, pstate, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) return pillar, errors def _external_pillar_data(self, pillar, val, key): - ''' + """ Builds actual pillar data structure and updates the ``pillar`` variable - ''' + """ ext = None args = salt.utils.args.get_function_argspec(self.ext_pillars[key]).args if isinstance(val, dict): - if ('extra_minion_data' in args) and self.extra_minion_data: + if ("extra_minion_data" in args) and self.extra_minion_data: ext = self.ext_pillars[key]( - self.minion_id, pillar, - extra_minion_data=self.extra_minion_data, **val) + self.minion_id, + pillar, + extra_minion_data=self.extra_minion_data, + **val + ) else: ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - if ('extra_minion_data' in args) and self.extra_minion_data: + if ("extra_minion_data" in args) and self.extra_minion_data: ext = self.ext_pillars[key]( - self.minion_id, pillar, *val, - extra_minion_data=self.extra_minion_data) + self.minion_id, + pillar, + *val, + extra_minion_data=self.extra_minion_data + ) else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - *val) + ext = self.ext_pillars[key](self.minion_id, pillar, *val) else: - if ('extra_minion_data' in args) and self.extra_minion_data: + if ("extra_minion_data" in args) and self.extra_minion_data: ext = self.ext_pillars[key]( self.minion_id, pillar, val, - extra_minion_data=self.extra_minion_data) + extra_minion_data=self.extra_minion_data, + ) else: - ext = self.ext_pillars[key](self.minion_id, - pillar, - val) + ext = self.ext_pillars[key](self.minion_id, pillar, val) return ext def ext_pillar(self, pillar, errors=None): - ''' + """ Render the external pillar data - ''' + """ if errors is None: errors = [] try: # Make sure that on-demand git_pillar is fetched before we try to # compile the pillar data. git_pillar will fetch a remote when # the git ext_pillar() func is run, but only for masterless. - if self.ext and 'git' in self.ext \ - and self.opts.get('__role') != 'minion': + if self.ext and "git" in self.ext and self.opts.get("__role") != "minion": # Avoid circular import import salt.utils.gitfs import salt.pillar.git_pillar + git_pillar = salt.utils.gitfs.GitPillar( self.opts, - self.ext['git'], + self.ext["git"], per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY, - global_only=salt.pillar.git_pillar.GLOBAL_ONLY) + global_only=salt.pillar.git_pillar.GLOBAL_ONLY, + ) git_pillar.fetch_remotes() except TypeError: # Handle malformed ext_pillar pass - if 'ext_pillar' not in self.opts: + if "ext_pillar" not in self.opts: return pillar, errors - if not isinstance(self.opts['ext_pillar'], list): + if not isinstance(self.opts["ext_pillar"], list): errors.append('The "ext_pillar" option is malformed') log.critical(errors[-1]) return pillar, errors @@ -959,65 +1095,63 @@ def ext_pillar(self, pillar, errors=None): pillar, self.pillar_override, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) - for run in self.opts['ext_pillar']: + for run in self.opts["ext_pillar"]: if not isinstance(run, dict): errors.append('The "ext_pillar" option is malformed') log.critical(errors[-1]) return {}, errors - if next(six.iterkeys(run)) in self.opts.get('exclude_ext_pillar', []): + if next(six.iterkeys(run)) in self.opts.get("exclude_ext_pillar", []): continue for key, val in six.iteritems(run): if key not in self.ext_pillars: log.critical( - 'Specified ext_pillar interface %s is unavailable', - key + "Specified ext_pillar interface %s is unavailable", key ) continue try: - ext = self._external_pillar_data(pillar, - val, - key) + ext = self._external_pillar_data(pillar, val, key) except Exception as exc: # pylint: disable=broad-except errors.append( - 'Failed to load ext_pillar {0}: {1}'.format( - key, - exc.__str__(), - ) + "Failed to load ext_pillar {0}: {1}".format(key, exc.__str__(),) ) log.error( - 'Exception caught loading ext_pillar \'%s\':\n%s', - key, ''.join(traceback.format_tb(sys.exc_info()[2])) + "Exception caught loading ext_pillar '%s':\n%s", + key, + "".join(traceback.format_tb(sys.exc_info()[2])), ) if ext: pillar = merge( pillar, ext, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) ext = None return pillar, errors def compile_pillar(self, ext=True): - ''' + """ Render the pillar data and return - ''' + """ top, top_errors = self.get_top() if ext: - if self.opts.get('ext_pillar_first', False): - self.opts['pillar'], errors = self.ext_pillar(self.pillar_override) + if self.opts.get("ext_pillar_first", False): + self.opts["pillar"], errors = self.ext_pillar(self.pillar_override) self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top, reload=True) pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge( - self.opts['pillar'], + self.opts["pillar"], pillar, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -1026,58 +1160,62 @@ def compile_pillar(self, ext=True): matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) errors.extend(top_errors) - if self.opts.get('pillar_opts', False): + if self.opts.get("pillar_opts", False): mopts = dict(self.opts) - if 'grains' in mopts: - mopts.pop('grains') - mopts['saltversion'] = __version__ - pillar['master'] = mopts - if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False): + if "grains" in mopts: + mopts.pop("grains") + mopts["saltversion"] = __version__ + pillar["master"] = mopts + if "pillar" in self.opts and self.opts.get("ssh_merge_pillar", False): pillar = merge( - self.opts['pillar'], + self.opts["pillar"], pillar, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) if errors: for error in errors: - log.critical('Pillar render error: %s', error) - pillar['_errors'] = errors + log.critical("Pillar render error: %s", error) + pillar["_errors"] = errors if self.pillar_override: pillar = merge( pillar, self.pillar_override, self.merge_strategy, - self.opts.get('renderer', 'yaml'), - self.opts.get('pillar_merge_lists', False)) + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: - pillar.setdefault('_errors', []).extend(decrypt_errors) + pillar.setdefault("_errors", []).extend(decrypt_errors) return pillar def decrypt_pillar(self, pillar): - ''' + """ Decrypt the specified pillar dictionary items, if configured to do so - ''' + """ errors = [] - if self.opts.get('decrypt_pillar'): - decrypt_pillar = self.opts['decrypt_pillar'] + if self.opts.get("decrypt_pillar"): + decrypt_pillar = self.opts["decrypt_pillar"] if not isinstance(decrypt_pillar, dict): - decrypt_pillar = \ - salt.utils.data.repack_dictlist(self.opts['decrypt_pillar']) + decrypt_pillar = salt.utils.data.repack_dictlist( + self.opts["decrypt_pillar"] + ) if not decrypt_pillar: - errors.append('decrypt_pillar config option is malformed') + errors.append("decrypt_pillar config option is malformed") for key, rend in six.iteritems(decrypt_pillar): ptr = salt.utils.data.traverse_dict( pillar, key, default=None, - delimiter=self.opts['decrypt_pillar_delimiter']) + delimiter=self.opts["decrypt_pillar_delimiter"], + ) if ptr is None: - log.debug('Pillar key %s not present', key) + log.debug("Pillar key %s not present", key) continue try: hash(ptr) @@ -1087,17 +1225,19 @@ def decrypt_pillar(self, pillar): try: ret = salt.utils.crypt.decrypt( ptr, - rend or self.opts['decrypt_pillar_default'], + rend or self.opts["decrypt_pillar_default"], renderers=self.rend, opts=self.opts, - valid_rend=self.opts['decrypt_pillar_renderers']) + valid_rend=self.opts["decrypt_pillar_renderers"], + ) if immutable: # Since the key pointed to an immutable type, we need # to replace it in the pillar dict. First we will find # the parent, and then we will replace the child key # with the return data from the renderer. parent, _, child = key.rpartition( - self.opts['decrypt_pillar_delimiter']) + self.opts["decrypt_pillar_delimiter"] + ) if not parent: # key is a top-level key, so the pointer to the # parent is the pillar dict itself. @@ -1107,21 +1247,20 @@ def decrypt_pillar(self, pillar): pillar, parent, default=None, - delimiter=self.opts['decrypt_pillar_delimiter']) + delimiter=self.opts["decrypt_pillar_delimiter"], + ) if ptr is not None: ptr[child] = ret except Exception as exc: # pylint: disable=broad-except - msg = 'Failed to decrypt pillar key \'{0}\': {1}'.format( - key, exc - ) + msg = "Failed to decrypt pillar key '{0}': {1}".format(key, exc) errors.append(msg) log.error(msg, exc_info=True) return errors def destroy(self): - ''' + """ This method exist in order to be API compatible with RemotePillar - ''' + """ if self._closing: return self._closing = True @@ -1129,6 +1268,7 @@ def destroy(self): # pylint: disable=W1701 def __del__(self): self.destroy() + # pylint: enable=W1701 diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 6903ccd..42ec985 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" Use Consul K/V as a Pillar source with values parsed as YAML :depends: - python-consul @@ -134,110 +134,111 @@ ext_pillar: - consul: my_consul_config expand_keys=false -''' +""" from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging import re -from salt.exceptions import CommandExecutionError -from salt.utils.dictupdate import update as dict_merge import salt.utils.minions import salt.utils.yaml +from salt.exceptions import CommandExecutionError +from salt.utils.dictupdate import update as dict_merge # Import third party libs try: import consul - if not hasattr(consul, '__version__'): - consul.__version__ = '0.1' # Some packages has no version, and so this pillar crashes on access to it. + + if not hasattr(consul, "__version__"): + consul.__version__ = "0.1" # Some packages has no version, and so this pillar crashes on access to it. except ImportError: consul = None -__virtualname__ = 'consul' +__virtualname__ = "consul" # Set up logging log = logging.getLogger(__name__) def __virtual__(): - ''' + """ Only return if python-consul is installed - ''' + """ return __virtualname__ if consul is not None else False -def ext_pillar(minion_id, - pillar, # pylint: disable=W0613 - conf): - ''' +def ext_pillar(minion_id, pillar, conf): # pylint: disable=W0613 + """ Check consul for all data - ''' + """ opts = {} temp = conf target_re = re.compile('target="(.*?)"') match = target_re.search(temp) if match: - opts['target'] = match.group(1) - temp = temp.replace(match.group(0), '') + opts["target"] = match.group(1) + temp = temp.replace(match.group(0), "") checker = salt.utils.minions.CkMinions(__opts__) - _res = checker.check_minions(opts['target'], 'compound') - minions = _res['minions'] - log.debug('Targeted minions: %r', minions) + _res = checker.check_minions(opts["target"], "compound") + minions = _res["minions"] + log.debug("Targeted minions: %r", minions) if minion_id not in minions: return {} - root_re = re.compile('(? Date: Mon, 9 Mar 2020 18:01:44 -0300 Subject: [PATCH 636/769] Fix bad RST in consul_pillar docstring --- src/saltext/consul/pillar/consul_pillar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 42ec985..fd68ee4 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -22,9 +22,9 @@ The ``consul.token`` requires python-consul >= 0.4.7. -If you have a multi-datacenter Consul cluster you can map your ``pillarenv``s -to your data centers by providing a dictionary of mappings in ``consul.dc`` -field: +If you have a multi-datacenter Consul cluster you can map your ``pillarenv`` +entries to your data centers by providing a dictionary of mappings in +``consul.dc`` field: .. code-block:: yaml From 0f4394906867a7cf586655ef3de559386cf7abc3 Mon Sep 17 00:00:00 2001 From: Matt Phillips Date: Tue, 21 Apr 2020 12:38:39 -0600 Subject: [PATCH 637/769] silence some high-noise log messages in pillar cache on larger installations these tend to fill all of the logs. lets drop the level of them since they aren't very useful. --- src/saltext/consul/pillar/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index c411b66..273aeb8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -65,7 +65,6 @@ def get_pillar( # If local pillar and we're caching, run through the cache system first log.debug("Determining pillar cache") if opts["pillar_cache"]: - log.info("Compiling pillar from cache") log.debug("get_pillar using pillar cache with ext: %s", ext) return PillarCache( opts, From 6874a4e45cd72da57362020f8d3ea25a16370a18 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 11 Mar 2019 16:41:32 -0500 Subject: [PATCH 638/769] Port #52156 to master Allow pillar relative includes Fixes #8875 and Fixes #56186 --- src/saltext/consul/pillar/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 273aeb8..9d0d145 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -814,7 +814,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): defaults = {} err = "" errors = [] - fn_ = self.client.get_state(sls, saltenv).get("dest", False) + state_data = self.client.get_state(sls, saltenv) + fn_ = state_data.get("dest", False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug( @@ -907,6 +908,16 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): self.avail[saltenv], sub_sls.lstrip(".").replace("/", "."), ) + if sub_sls.startswith("."): + if state_data.get("source", "").endswith( + "/init.sls" + ): + include_parts = sls.split(".") + else: + include_parts = sls.split(".")[:-1] + sub_sls = ".".join(include_parts + [sub_sls[1:]]) + matches = fnmatch.filter(self.avail[saltenv], sub_sls,) + matched_pstates.extend(matches) except KeyError: errors.extend( [ From f77914378ad563c3258bc8ce60b00d41367f01ee Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Mon, 11 Mar 2019 16:41:32 -0500 Subject: [PATCH 639/769] Allow pillar relative includes Fixes #8875 - a longstanding issue --- src/saltext/consul/pillar/__init__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 273aeb8..ab16c1b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -814,7 +814,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): defaults = {} err = "" errors = [] - fn_ = self.client.get_state(sls, saltenv).get("dest", False) + state_data = self.client.get_state(sls, saltenv) + fn_ = state_data.get("dest", False) if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug( @@ -903,10 +904,16 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): else: key = None try: - matched_pstates = fnmatch.filter( - self.avail[saltenv], - sub_sls.lstrip(".").replace("/", "."), - ) + if sub_sls.startswith("."): + if state_data.get("source", "").endswith( + "/init.sls" + ): + include_parts = sls.split(".") + else: + include_parts = sls.split(".")[:-1] + sub_sls = ".".join(include_parts + [sub_sls[1:]]) + matches = fnmatch.filter(self.avail[saltenv], sub_sls,) + matched_pstates.extend(matches) except KeyError: errors.extend( [ From 0b6d68a39abb6b7ffc58a910c27e6efc24b5a8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 27 Dec 2018 14:30:08 +0100 Subject: [PATCH 640/769] Bugfix: setting empty 'webhook' option in test_no_webhook From f819b6096c104980a79def7e1850d7609f28d97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Thu, 27 Dec 2018 14:08:31 +0100 Subject: [PATCH 641/769] Bugfix: checking webhook is empty or None From c373d87b47ddfa3042796580ef83dfd38d3163f5 Mon Sep 17 00:00:00 2001 From: lnattrass Date: Mon, 11 May 2020 12:11:12 -0400 Subject: [PATCH 642/769] Fix: include of missing pillars silently continues This will cause render_pstate to return an error when the pillar is not found. Currently, missing pillars are silently continued, which may lead to unexpected pillar states (and makes testing the existence of pillars difficult.) --- src/saltext/consul/pillar/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 9d0d145..78b0cc9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -926,6 +926,9 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): ] ) matched_pstates = [sub_sls] + # If matched_pstates is empty, set to sub_sls + if len(matched_pstates) < 1: + matched_pstates = [sub_sls] for m_sub_sls in matched_pstates: if m_sub_sls not in mods: nstate, mods, err = self.render_pstate( From 4aaa1ab3765cdb2c2d6a686b975a5efb1e244322 Mon Sep 17 00:00:00 2001 From: Wesley Whetstone Date: Mon, 18 May 2020 15:44:48 -0700 Subject: [PATCH 643/769] add plist to serializers doc index --- docs/ref/cache/all/salt.cache.consul.rst | 3 ++- docs/ref/modules/all/salt.modules.consul.rst | 2 +- docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 7 ++++--- docs/ref/sdb/all/salt.sdb.consul.rst | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst index d28bcc0..85386cc 100644 --- a/docs/ref/cache/all/salt.cache.consul.rst +++ b/docs/ref/cache/all/salt.cache.consul.rst @@ -1,5 +1,6 @@ +================= salt.cache.consul ================= .. automodule:: salt.cache.consul - :members: + :members: \ No newline at end of file diff --git a/docs/ref/modules/all/salt.modules.consul.rst b/docs/ref/modules/all/salt.modules.consul.rst index 4567678..f6783d5 100644 --- a/docs/ref/modules/all/salt.modules.consul.rst +++ b/docs/ref/modules/all/salt.modules.consul.rst @@ -3,4 +3,4 @@ salt.modules.consul =================== .. automodule:: salt.modules.consul - :members: + :members: \ No newline at end of file diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst index c27449b..523ff0f 100644 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -1,5 +1,6 @@ -salt.pillar.consul_pillar module -================================ +========================= +salt.pillar.consul_pillar +========================= .. automodule:: salt.pillar.consul_pillar - :members: + :members: \ No newline at end of file diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst index 5a47d2b..cf4ade2 100644 --- a/docs/ref/sdb/all/salt.sdb.consul.rst +++ b/docs/ref/sdb/all/salt.sdb.consul.rst @@ -1,5 +1,6 @@ -salt.sdb.consul module -====================== +=============== +salt.sdb.consul +=============== .. automodule:: salt.sdb.consul - :members: + :members: \ No newline at end of file From 0db5b035608ad28fd8f22b641a07a220b2116db0 Mon Sep 17 00:00:00 2001 From: Wesley Whetstone Date: Mon, 18 May 2020 17:07:42 -0700 Subject: [PATCH 644/769] Revert "add plist to serializers doc index" This reverts commit 7512218a81358785b4a4337f4eb54b7db67cc0a7. --- docs/ref/cache/all/salt.cache.consul.rst | 3 +-- docs/ref/modules/all/salt.modules.consul.rst | 2 +- docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 7 +++---- docs/ref/sdb/all/salt.sdb.consul.rst | 7 +++---- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/ref/cache/all/salt.cache.consul.rst b/docs/ref/cache/all/salt.cache.consul.rst index 85386cc..d28bcc0 100644 --- a/docs/ref/cache/all/salt.cache.consul.rst +++ b/docs/ref/cache/all/salt.cache.consul.rst @@ -1,6 +1,5 @@ -================= salt.cache.consul ================= .. automodule:: salt.cache.consul - :members: \ No newline at end of file + :members: diff --git a/docs/ref/modules/all/salt.modules.consul.rst b/docs/ref/modules/all/salt.modules.consul.rst index f6783d5..4567678 100644 --- a/docs/ref/modules/all/salt.modules.consul.rst +++ b/docs/ref/modules/all/salt.modules.consul.rst @@ -3,4 +3,4 @@ salt.modules.consul =================== .. automodule:: salt.modules.consul - :members: \ No newline at end of file + :members: diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst index 523ff0f..c27449b 100644 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -1,6 +1,5 @@ -========================= -salt.pillar.consul_pillar -========================= +salt.pillar.consul_pillar module +================================ .. automodule:: salt.pillar.consul_pillar - :members: \ No newline at end of file + :members: diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst index cf4ade2..5a47d2b 100644 --- a/docs/ref/sdb/all/salt.sdb.consul.rst +++ b/docs/ref/sdb/all/salt.sdb.consul.rst @@ -1,6 +1,5 @@ -=============== -salt.sdb.consul -=============== +salt.sdb.consul module +====================== .. automodule:: salt.sdb.consul - :members: \ No newline at end of file + :members: From 8d0d21a9afeb2af93c43f3fccfcff9d84d634105 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Sat, 11 Apr 2020 21:51:39 -0600 Subject: [PATCH 645/769] Use file_client=remote for pillar when file_client=local but use_master_when_local is true --- src/saltext/consul/pillar/__init__.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 78b0cc9..1987c6b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -58,9 +58,22 @@ def get_pillar( """ Return the correct pillar driver based on the file_client option """ + # When file_client is 'local' this makes the minion masterless + # but sometimes we want the minion to read its files from the local + # filesystem instead of asking for them from the master, but still + # get commands from the master. + # To enable this functionality set file_client=local and + # use_master_when_local=True in the minion config. Then here we override + # the file client to be 'remote' for getting pillar. If we don't do this + # then the minion never sends the event that the master uses to update + # its minion_data_cache. If the master doesn't update the minion_data_cache + # then the SSE salt-master plugin won't see any grains for those minions. file_client = opts["file_client"] if opts.get("master_type") == "disable" and file_client == "remote": file_client = "local" + elif file_client == "local" and opts.get("use_master_when_local"): + file_client = "remote" + ptype = {"remote": RemotePillar, "local": Pillar}.get(file_client, Pillar) # If local pillar and we're caching, run through the cache system first log.debug("Determining pillar cache") @@ -488,7 +501,9 @@ def __init__( self.client = salt.fileclient.get_file_client(self.opts, True) self.avail = self.__gather_avail() - if opts.get("file_client", "") == "local": + if opts.get("file_client", "") == "local" and not opts.get( + "use_master_when_local", False + ): opts["grains"] = grains # if we didn't pass in functions, lets load them @@ -1204,7 +1219,6 @@ def compile_pillar(self, ext=True): decrypt_errors = self.decrypt_pillar(pillar) if decrypt_errors: pillar.setdefault("_errors", []).extend(decrypt_errors) - return pillar def decrypt_pillar(self, pillar): From 9964807ee0c1c78e4c0364d8ea36d79998ab7293 Mon Sep 17 00:00:00 2001 From: "C. R. Oldham" Date: Sun, 12 Apr 2020 15:35:34 -0600 Subject: [PATCH 646/769] Blacken --- src/saltext/consul/pillar/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 1987c6b..12d609a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -120,6 +120,8 @@ def get_async_pillar( file_client = opts["file_client"] if opts.get("master_type") == "disable" and file_client == "remote": file_client = "local" + elif file_client == "local" and opts.get("use_master_when_local"): + file_client = "remote" ptype = {"remote": AsyncRemotePillar, "local": AsyncPillar}.get( file_client, AsyncPillar ) From 8ad6a188697498e130d7d17c1d64144b66d9809e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 21 May 2020 21:45:51 +0100 Subject: [PATCH 647/769] Better name for the pytest only directory --- tests/pytests/unit/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/pytests/unit/__init__.py diff --git a/tests/pytests/unit/__init__.py b/tests/pytests/unit/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/pytests/unit/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From e32f41e3a55e91fa5412c7ddb06daa5fea4795fd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 7 Jul 2020 17:42:00 +0100 Subject: [PATCH 648/769] Remove Sodium related deprecation warnings Fixes #57516 Additionally, under Python 2, show and error when salt is imported and exit on CLI usage. --- src/saltext/consul/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index c620fb5..36109fc 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -3,13 +3,18 @@ Salt package """ -# Import Python libs from __future__ import absolute_import, print_function, unicode_literals import importlib import sys import warnings +if sys.version_info < (3,): + sys.stderr.write( + "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" + ) + sys.stderr.flush() + class TornadoImporter(object): def find_module(self, module_name, package_path=None): From eb4c529eb98b328e0c890cc25f1f0edfe0d3fee3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 9 Jun 2020 07:40:53 +0100 Subject: [PATCH 649/769] Migrate `tests/unit/modules/test_alternatives.py` to PyTest --- tests/pytests/unit/modules/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/pytests/unit/modules/__init__.py diff --git a/tests/pytests/unit/modules/__init__.py b/tests/pytests/unit/modules/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/pytests/unit/modules/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From bbff53e012a5b0bcc40bd0aa82818b444ae07889 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 9 Jun 2020 07:43:42 +0100 Subject: [PATCH 650/769] Migrate `tests/unit/states/test_alternatives.py` to PyTest --- tests/pytests/unit/states/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/pytests/unit/states/__init__.py diff --git a/tests/pytests/unit/states/__init__.py b/tests/pytests/unit/states/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/pytests/unit/states/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From 9ca20f20d7f8d6b99c084a7f2d3edaf05263c9d2 Mon Sep 17 00:00:00 2001 From: Jean-Yves NOLEN Date: Tue, 9 Jun 2020 13:02:04 +0200 Subject: [PATCH 651/769] Filterwarning overide previously defined filters ISSUE:57582 --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 36109fc..40d7a8e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -38,6 +38,7 @@ def load_module(self, name): "", # No deprecation message match DeprecationWarning, # This filter is for DeprecationWarnings r"^(salt|salt\.(.*))$", # Match module(s) 'salt' and 'salt.' + append=True, ) # While we are supporting Python2.6, hide nested with-statements warnings @@ -45,6 +46,7 @@ def load_module(self, name): "ignore", "With-statements now directly support multiple context managers", DeprecationWarning, + append=True, ) # Filter the backports package UserWarning about being re-imported @@ -52,6 +54,7 @@ def load_module(self, name): "ignore", "^Module backports was already imported from (.*), but (.*) is being added to sys.path$", UserWarning, + append=True, ) From 78dd88da0543cfaf1082e9a0e4bac595f02b6b92 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 20 Aug 2020 18:19:25 +0100 Subject: [PATCH 652/769] Switch virt integration tests to PyTest --- tests/pytests/integration/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/pytests/integration/conftest.py diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py new file mode 100644 index 0000000..9e341f4 --- /dev/null +++ b/tests/pytests/integration/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture(scope="package") +def salt_master(request, salt_factories): + return salt_factories.spawn_master(request, "master") From f234a8437f65adf0f40aaf98bd034032eaeb15c8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 31 Aug 2020 15:58:49 +0100 Subject: [PATCH 653/769] Import the pytest fixtures on the "main" conftest.py --- tests/pytests/integration/conftest.py | 47 ++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 9e341f4..a392b0b 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -2,5 +2,50 @@ @pytest.fixture(scope="package") -def salt_master(request, salt_factories): +def salt_master(request, salt_factories, salt_master_config): return salt_factories.spawn_master(request, "master") + + +@pytest.fixture(scope="package") +def salt_minion(request, salt_factories, salt_master, salt_minion_config): + proc = salt_factories.spawn_minion(request, "minion", master_id="master") + # Sync All + salt_call_cli = salt_factories.get_salt_call_cli("minion") + ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) + assert ret.exitcode == 0, ret + return proc + + +@pytest.fixture(scope="package") +def salt_sub_minion(request, salt_factories, salt_master, salt_sub_minion_config): + proc = salt_factories.spawn_minion(request, "sub_minion", master_id="master") + # Sync All + salt_call_cli = salt_factories.get_salt_call_cli("sub_minion") + ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) + assert ret.exitcode == 0, ret + return proc + + +@pytest.fixture(scope="package") +def salt_cli(salt_factories, salt_minion, salt_master): + return salt_factories.get_salt_cli(salt_master.config["id"]) + + +@pytest.fixture(scope="package") +def salt_cp_cli(salt_factories, salt_minion, salt_master): + return salt_factories.get_salt_cp_cli(salt_master.config["id"]) + + +@pytest.fixture(scope="package") +def salt_key_cli(salt_factories, salt_minion, salt_master): + return salt_factories.get_salt_key_cli(salt_master.config["id"]) + + +@pytest.fixture(scope="package") +def salt_run_cli(salt_factories, salt_minion, salt_master): + return salt_factories.get_salt_run_cli(salt_master.config["id"]) + + +@pytest.fixture(scope="package") +def salt_call_cli(salt_factories, salt_minion, salt_master): + return salt_factories.get_salt_call_cli(salt_minion.config["id"]) From baca2924866d039bc9dcb757e7a543bbf5a077f4 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 24 Aug 2020 14:57:00 -0700 Subject: [PATCH 654/769] Fixing pillar caching when pillar environments are involved. adding a test. --- src/saltext/consul/pillar/__init__.py | 81 +++++++++++++-------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 12d609a..3dba4dd 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- """ Render the pillar data """ # Import python libs -from __future__ import absolute_import, print_function, unicode_literals import collections import copy @@ -138,7 +136,7 @@ def get_async_pillar( ) -class RemotePillarMixin(object): +class RemotePillarMixin: """ Common remote pillar functionality """ @@ -255,7 +253,7 @@ def compile_pillar(self): if not isinstance(ret_pillar, dict): msg = ( - "Got a bad pillar from master, type {0}, expecting dict: " "{1}" + "Got a bad pillar from master, type {}, expecting dict: " "{}" ).format(type(ret_pillar).__name__, ret_pillar) log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! @@ -360,7 +358,7 @@ def __del__(self): # pylint: enable=W1701 -class PillarCache(object): +class PillarCache: """ Return a cached pillar if it exists, otherwise cache it. @@ -459,7 +457,11 @@ def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs else: # We found the minion but not the env. Store it. fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id][self.pillarenv] = fresh_pillar + + minion_cache = self.cache[self.minion_id] + minion_cache[self.pillarenv] = fresh_pillar + self.cache[self.minion_id] = minion_cache + log.debug( "Pillar cache miss for pillarenv %s for minion %s", self.pillarenv, @@ -475,7 +477,7 @@ def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs return fresh_pillar -class Pillar(object): +class Pillar: """ Read over the pillar top files and render the pillar data """ @@ -551,7 +553,7 @@ def __valid_on_demand_ext_pillar(self, opts): on_demand = opts.get("on_demand_ext_pillar", []) try: - invalid_on_demand = set([x for x in self.ext if x not in on_demand]) + invalid_on_demand = {x for x in self.ext if x not in on_demand} except TypeError: # Prevent traceback when on_demand_ext_pillar option is malformed log.error( @@ -628,7 +630,7 @@ def _get_envs(self): """ Pull the file server environments out of the master options """ - envs = set(["base"]) + envs = {"base"} if "pillar_roots" in self.opts: envs.update(list(self.opts["pillar_roots"])) return envs @@ -659,7 +661,7 @@ def get_tops(self): else: saltenvs = self._get_envs() if self.opts.get("pillar_source_merging_strategy", None) == "none": - saltenvs &= set([self.saltenv or "base"]) + saltenvs &= {self.saltenv or "base"} for saltenv in saltenvs: top = self.client.cache_file(self.opts["state_top"], saltenv) @@ -677,12 +679,12 @@ def get_tops(self): ) except Exception as exc: # pylint: disable=broad-except errors.append( - ("Rendering Primary Top file failed, render error:\n{0}".format(exc)) + "Rendering Primary Top file failed, render error:\n{}".format(exc) ) log.exception("Pillar rendering failed for minion %s", self.minion_id) # Search initial top files for includes - for saltenv, ctops in six.iteritems(tops): + for saltenv, ctops in tops.items(): for ctop in ctops: if "include" not in ctop: continue @@ -692,7 +694,7 @@ def get_tops(self): # Go through the includes and pull out the extra tops and add them while include: pops = [] - for saltenv, states in six.iteritems(include): + for saltenv, states in include.items(): pops.append(saltenv) if not states: continue @@ -714,7 +716,7 @@ def get_tops(self): except Exception as exc: # pylint: disable=broad-except errors.append( ( - "Rendering Top file {0} failed, render error" ":\n{1}" + "Rendering Top file {} failed, render error" ":\n{}" ).format(sls, exc) ) done[saltenv].append(sls) @@ -730,9 +732,9 @@ def merge_tops(self, tops): """ top = collections.defaultdict(OrderedDict) orders = collections.defaultdict(OrderedDict) - for ctops in six.itervalues(tops): + for ctops in tops.values(): for ctop in ctops: - for saltenv, targets in six.iteritems(ctop): + for saltenv, targets in ctop.items(): if saltenv == "include": continue for tgt in targets: @@ -754,7 +756,7 @@ def merge_tops(self, tops): orders[saltenv][tgt] = order if comp.get("ignore_missing", False): ignore_missing = True - if isinstance(comp, six.string_types): + if isinstance(comp, str): states[comp] = True if ignore_missing: if saltenv not in self.ignored_pillars: @@ -770,7 +772,7 @@ def sort_top_targets(self, top, orders): """ sorted_top = collections.defaultdict(OrderedDict) # pylint: disable=cell-var-from-loop - for saltenv, targets in six.iteritems(top): + for saltenv, targets in top.items(): sorted_targets = sorted(targets, key=lambda target: orders[saltenv][target]) for target in sorted_targets: sorted_top[saltenv][target] = targets[target] @@ -803,11 +805,11 @@ def top_matches(self, top, reload=False): matches = {} if reload: self.matchers = salt.loader.matchers(self.opts) - for saltenv, body in six.iteritems(top): + for saltenv, body in top.items(): if self.opts["pillarenv"]: if saltenv != self.opts["pillarenv"]: continue - for match, data in six.iteritems(body): + for match, data in body.items(): if self.matchers["confirm_top.confirm_top"]( match, data, self.opts.get("nodegroups", {}), ): @@ -816,10 +818,7 @@ def top_matches(self, top, reload=False): else: env_matches = matches[saltenv] for item in data: - if ( - isinstance(item, six.string_types) - and item not in env_matches - ): + if isinstance(item, str) and item not in env_matches: env_matches.append(item) return matches @@ -843,14 +842,14 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): return None, mods, errors elif self.opts["pillar_roots"].get(saltenv): msg = ( - "Specified SLS '{0}' in environment '{1}' is not" + "Specified SLS '{}' in environment '{}' is not" " available on the salt master" ).format(sls, saltenv) log.error(msg) errors.append(msg) else: msg = ( - "Specified SLS '{0}' in environment '{1}' was not " + "Specified SLS '{}' in environment '{}' was not " "found. ".format(sls, saltenv) ) if self.opts.get("__git_pillar", False) is True: @@ -885,11 +884,11 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): **defaults ) except Exception as exc: # pylint: disable=broad-except - msg = "Rendering SLS '{0}' failed, render error:\n{1}".format(sls, exc) + msg = "Rendering SLS '{}' failed, render error:\n{}".format(sls, exc) log.critical(msg, exc_info=True) if self.opts.get("pillar_safe_render_error", True): errors.append( - "Rendering SLS '{0}' failed. Please see master log for " + "Rendering SLS '{}' failed. Please see master log for " "details.".format(sls) ) else: @@ -898,14 +897,14 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate = None if state: if not isinstance(state, dict): - msg = "SLS '{0}' does not render to a dictionary".format(sls) + msg = "SLS '{}' does not render to a dictionary".format(sls) log.error(msg) errors.append(msg) else: if "include" in state: if not isinstance(state["include"], list): msg = ( - "Include Declaration in SLS '{0}' is not " + "Include Declaration in SLS '{}' is not " "formed as a list".format(sls) ) log.error(msg) @@ -915,7 +914,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): include_states = [] for sub_sls in state.pop("include"): if isinstance(sub_sls, dict): - sub_sls, v = next(six.iteritems(sub_sls)) + sub_sls, v = next(iter(sub_sls.items())) defaults = v.get("defaults", {}) key = v.get("key", None) else: @@ -939,7 +938,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): errors.extend( [ "No matching pillar environment for environment " - "'{0}' found".format(saltenv) + "'{}' found".format(saltenv) ] ) matched_pstates = [sub_sls] @@ -1001,7 +1000,7 @@ def render_pillar(self, matches, errors=None): pillar = copy.copy(self.pillar_override) if errors is None: errors = [] - for saltenv, pstates in six.iteritems(matches): + for saltenv, pstates in matches.items(): pstatefiles = [] mods = set() for sls_match in pstates: @@ -1012,7 +1011,7 @@ def render_pillar(self, matches, errors=None): errors.extend( [ "No matching pillar environment for environment " - "'{0}' found".format(saltenv) + "'{}' found".format(saltenv) ] ) if matched_pstates: @@ -1034,7 +1033,7 @@ def render_pillar(self, matches, errors=None): "a sign of a malformed pillar sls file. Returned " "errors: %s", sls, - ", ".join(["'{0}'".format(e) for e in errors]), + ", ".join(["'{}'".format(e) for e in errors]), ) continue pillar = merge( @@ -1134,9 +1133,9 @@ def ext_pillar(self, pillar, errors=None): errors.append('The "ext_pillar" option is malformed') log.critical(errors[-1]) return {}, errors - if next(six.iterkeys(run)) in self.opts.get("exclude_ext_pillar", []): + if next(iter(run.keys())) in self.opts.get("exclude_ext_pillar", []): continue - for key, val in six.iteritems(run): + for key, val in run.items(): if key not in self.ext_pillars: log.critical( "Specified ext_pillar interface %s is unavailable", key @@ -1146,7 +1145,7 @@ def ext_pillar(self, pillar, errors=None): ext = self._external_pillar_data(pillar, val, key) except Exception as exc: # pylint: disable=broad-except errors.append( - "Failed to load ext_pillar {0}: {1}".format(key, exc.__str__(),) + "Failed to load ext_pillar {}: {}".format(key, exc.__str__(),) ) log.error( "Exception caught loading ext_pillar '%s':\n%s", @@ -1236,7 +1235,7 @@ def decrypt_pillar(self, pillar): ) if not decrypt_pillar: errors.append("decrypt_pillar config option is malformed") - for key, rend in six.iteritems(decrypt_pillar): + for key, rend in decrypt_pillar.items(): ptr = salt.utils.data.traverse_dict( pillar, key, @@ -1281,7 +1280,7 @@ def decrypt_pillar(self, pillar): if ptr is not None: ptr[child] = ret except Exception as exc: # pylint: disable=broad-except - msg = "Failed to decrypt pillar key '{0}': {1}".format(key, exc) + msg = "Failed to decrypt pillar key '{}': {}".format(key, exc) errors.append(msg) log.error(msg, exc_info=True) return errors @@ -1306,5 +1305,5 @@ def __del__(self): class AsyncPillar(Pillar): @salt.ext.tornado.gen.coroutine def compile_pillar(self, ext=True): - ret = super(AsyncPillar, self).compile_pillar(ext=ext) + ret = super().compile_pillar(ext=ext) raise salt.ext.tornado.gen.Return(ret) From a0c9b36c4dfbe0623a6411e422bac44af023fce8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 3 Sep 2020 09:58:54 +0100 Subject: [PATCH 655/769] Bump pytest-salt-factories to 0.92.x --- tests/pytests/integration/conftest.py | 109 ++++++++++++++++++++------ 1 file changed, 83 insertions(+), 26 deletions(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index a392b0b..b770c7c 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -1,51 +1,108 @@ +""" + tests.pytests.integration.conftest + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + PyTest fixtures +""" import pytest @pytest.fixture(scope="package") -def salt_master(request, salt_factories, salt_master_config): - return salt_factories.spawn_master(request, "master") +def salt_master(salt_master_factory): + """ + A running salt-master fixture + """ + with salt_master_factory.started(): + yield salt_master_factory + + +@pytest.fixture(scope="package") +def salt_minion(salt_master, salt_minion_factory): + """ + A running salt-minion fixture + """ + assert salt_master.is_running() + with salt_minion_factory.started(): + # Sync All + salt_call_cli = salt_minion_factory.get_salt_call_cli() + ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) + assert ret.exitcode == 0, ret + yield salt_minion_factory + + +@pytest.fixture(scope="module") +def salt_sub_minion(salt_master, salt_sub_minion_factory): + """ + A second running salt-minion fixture + """ + assert salt_master.is_running() + with salt_sub_minion_factory.started(): + # Sync All + salt_call_cli = salt_sub_minion_factory.get_salt_call_cli() + ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) + assert ret.exitcode == 0, ret + yield salt_sub_minion_factory @pytest.fixture(scope="package") -def salt_minion(request, salt_factories, salt_master, salt_minion_config): - proc = salt_factories.spawn_minion(request, "minion", master_id="master") - # Sync All - salt_call_cli = salt_factories.get_salt_call_cli("minion") - ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.exitcode == 0, ret - return proc +def salt_proxy(salt_master, salt_proxy_factory): + """ + A running salt-proxy fixture + """ + assert salt_master.is_running() + with salt_proxy_factory.started(): + yield salt_proxy_factory @pytest.fixture(scope="package") -def salt_sub_minion(request, salt_factories, salt_master, salt_sub_minion_config): - proc = salt_factories.spawn_minion(request, "sub_minion", master_id="master") - # Sync All - salt_call_cli = salt_factories.get_salt_call_cli("sub_minion") - ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.exitcode == 0, ret - return proc +def salt_cli(salt_master): + """ + The ``salt`` CLI as a fixture against the running master + """ + assert salt_master.is_running() + return salt_master.get_salt_cli() @pytest.fixture(scope="package") -def salt_cli(salt_factories, salt_minion, salt_master): - return salt_factories.get_salt_cli(salt_master.config["id"]) +def salt_call_cli(salt_minion): + """ + The ``salt-call`` CLI as a fixture against the running minion + """ + assert salt_minion.is_running() + return salt_minion.get_salt_call_cli() @pytest.fixture(scope="package") -def salt_cp_cli(salt_factories, salt_minion, salt_master): - return salt_factories.get_salt_cp_cli(salt_master.config["id"]) +def salt_cp_cli(salt_master): + """ + The ``salt-cp`` CLI as a fixture against the running master + """ + assert salt_master.is_running() + return salt_master.get_salt_cp_cli() @pytest.fixture(scope="package") -def salt_key_cli(salt_factories, salt_minion, salt_master): - return salt_factories.get_salt_key_cli(salt_master.config["id"]) +def salt_key_cli(salt_master): + """ + The ``salt-key`` CLI as a fixture against the running master + """ + assert salt_master.is_running() + return salt_master.get_salt_key_cli() @pytest.fixture(scope="package") -def salt_run_cli(salt_factories, salt_minion, salt_master): - return salt_factories.get_salt_run_cli(salt_master.config["id"]) +def salt_run_cli(salt_master): + """ + The ``salt-run`` CLI as a fixture against the running master + """ + assert salt_master.is_running() + return salt_master.get_salt_run_cli() @pytest.fixture(scope="package") -def salt_call_cli(salt_factories, salt_minion, salt_master): - return salt_factories.get_salt_call_cli(salt_minion.config["id"]) +def salt_ssh_cli(salt_master): + """ + The ``salt-ssh`` CLI as a fixture against the running master + """ + assert salt_master.is_running() + return salt_master.get_salt_ssh_cli() From cbcb9632d3f3d9cadadcf662ce116ecda34039dc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 8 Oct 2020 05:13:41 +0100 Subject: [PATCH 656/769] Fix and simplify `tests/pytests/functional/modules/test_opkg.py` --- tests/pytests/functional/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/pytests/functional/__init__.py diff --git a/tests/pytests/functional/__init__.py b/tests/pytests/functional/__init__.py new file mode 100644 index 0000000..e69de29 From 297bbad66aaaf84e2d53ab5220e5880f536be46b Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 2 Nov 2020 10:07:52 -0800 Subject: [PATCH 657/769] Log a different obeject when debugging if we're using disk cache vs memory cache. The disk cache pillar class has the _dict_ object but the cache pillar object which is used with the memory cache does not include a _dict obeject because it is a dict already. Adding a test when pillar_cache_backend is memory. --- src/saltext/consul/pillar/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 3dba4dd..57f4b4d 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -2,7 +2,6 @@ Render the pillar data """ -# Import python libs import collections import copy @@ -15,8 +14,6 @@ import salt.ext.tornado.gen import salt.fileclient - -# Import salt libs import salt.loader import salt.minion import salt.transport.client @@ -27,8 +24,6 @@ import salt.utils.dictupdate import salt.utils.url from salt.exceptions import SaltClientError - -# Import 3rd-party libs from salt.ext import six from salt.template import compile_template @@ -442,7 +437,12 @@ def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs self.minion_id, self.pillarenv, ) - log.debug("Scanning cache: %s", self.cache._dict) + if self.opts["pillar_cache_backend"] == "memory": + cache_dict = self.cache + else: + cache_dict = self.cache._dict + + log.debug("Scanning cache: %s", cache_dict) # Check the cache! if self.minion_id in self.cache: # Keyed by minion_id # TODO Compare grains, etc? @@ -473,7 +473,7 @@ def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs fresh_pillar = self.fetch_pillar() self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} log.debug("Pillar cache miss for minion %s", self.minion_id) - log.debug("Current pillar cache: %s", self.cache._dict) # FIXME hack! + log.debug("Current pillar cache: %s", cache_dict) # FIXME hack! return fresh_pillar From dafdf6ce53a9a82d72c1837fe8cefd0ed5da4989 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 30 Oct 2020 15:09:40 +0000 Subject: [PATCH 658/769] Drop Py2 and six usage --- src/saltext/consul/modules/consul.py | 171 +++++++++++++-------------- 1 file changed, 81 insertions(+), 90 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index a9f2c2e..ac1c7f6 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Interact with Consul @@ -6,21 +5,16 @@ """ -# Import Python Libs -from __future__ import absolute_import, print_function, unicode_literals import base64 +import http.client import logging +import urllib -# Import salt libs import salt.utils.http import salt.utils.json from salt.exceptions import SaltInvocationError -# Import 3rd-party libs -from salt.ext import six -from salt.ext.six.moves import http_client, urllib - log = logging.getLogger(__name__) @@ -75,7 +69,7 @@ def _query( token = _get_token() headers = {"X-Consul-Token": token, "Content-Type": "application/json"} - base_url = urllib.parse.urljoin(consul_url, "{0}/".format(api_version)) + base_url = urllib.parse.urljoin(consul_url, "{}/".format(api_version)) url = urllib.parse.urljoin(base_url, function, False) if method == "GET": @@ -96,12 +90,12 @@ def _query( opts=__opts__, ) - if result.get("status", None) == http_client.OK: + if result.get("status", None) == http.client.OK: ret["data"] = result.get("dict", result) ret["res"] = True - elif result.get("status", None) == http_client.NO_CONTENT: + elif result.get("status", None) == http.client.NO_CONTENT: ret["res"] = False - elif result.get("status", None) == http_client.NOT_FOUND: + elif result.get("status", None) == http.client.NOT_FOUND: ret["data"] = "Key not found." ret["res"] = False else: @@ -149,7 +143,7 @@ def list_(consul_url=None, token=None, key=None, **kwargs): query_params["recurse"] = "True" function = "kv/" else: - function = "kv/{0}".format(key) + function = "kv/{}".format(key) query_params["keys"] = "True" query_params["separator"] = "/" @@ -203,7 +197,7 @@ def get(consul_url=None, key=None, token=None, recurse=False, decode=False, raw= raise SaltInvocationError('Required argument "key" is missing.') query_params = {} - function = "kv/{0}".format(key) + function = "kv/{}".format(key) if recurse: query_params["recurse"] = "True" if raw: @@ -269,7 +263,7 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): for _l2 in conflicting_args: if _l1 in kwargs and _l2 in kwargs and _l1 != _l2: raise SaltInvocationError( - "Using arguments `{0}` and `{1}`" + "Using arguments `{}` and `{}`" " together is invalid.".format(_l1, _l2) ) @@ -285,14 +279,12 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if "cas" in kwargs: if _current["res"]: if kwargs["cas"] == 0: - ret["message"] = "Key {0} exists, index " "must be non-zero.".format( - key - ) + ret["message"] = "Key {} exists, index " "must be non-zero.".format(key) ret["res"] = False return ret if kwargs["cas"] != _current["data"]["ModifyIndex"]: - ret["message"] = "Key {0} exists, but indexes " "do not match.".format( + ret["message"] = "Key {} exists, but indexes " "do not match.".format( key ) ret["res"] = False @@ -301,13 +293,13 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): else: ret[ "message" - ] = "Key {0} does not exists, " "CAS argument can not be used.".format(key) + ] = "Key {} does not exists, " "CAS argument can not be used.".format(key) ret["res"] = False return ret if "acquire" in kwargs: if kwargs["acquire"] not in available_sessions: - ret["message"] = "{0} is not a valid session.".format(kwargs["acquire"]) + ret["message"] = "{} is not a valid session.".format(kwargs["acquire"]) ret["res"] = False return ret @@ -319,18 +311,18 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if _current["data"]["Session"] == kwargs["release"]: query_params["release"] = kwargs["release"] else: - ret["message"] = "{0} locked by another session.".format(key) + ret["message"] = "{} locked by another session.".format(key) ret["res"] = False return ret else: - ret["message"] = "{0} is not a valid session.".format(kwargs["acquire"]) + ret["message"] = "{} is not a valid session.".format(kwargs["acquire"]) ret["res"] = False else: log.error("Key {0} does not exist. Skipping release.") data = value - function = "kv/{0}".format(key) + function = "kv/{}".format(key) method = "PUT" ret = _query( consul_url=consul_url, @@ -343,10 +335,10 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if ret["res"]: ret["res"] = True - ret["data"] = "Added key {0} with value {1}.".format(key, value) + ret["data"] = "Added key {} with value {}.".format(key, value) else: ret["res"] = False - ret["data"] = "Unable to add key {0} with value {1}.".format(key, value) + ret["data"] = "Unable to add key {} with value {}.".format(key, value) return ret @@ -397,7 +389,7 @@ def delete(consul_url=None, token=None, key=None, **kwargs): ret["res"] = False return ret - function = "kv/{0}".format(key) + function = "kv/{}".format(key) ret = _query( consul_url=consul_url, token=token, @@ -408,10 +400,10 @@ def delete(consul_url=None, token=None, key=None, **kwargs): if ret["res"]: ret["res"] = True - ret["message"] = "Deleted key {0}.".format(key) + ret["message"] = "Deleted key {}.".format(key) else: ret["res"] = False - ret["message"] = "Unable to delete key {0}.".format(key) + ret["message"] = "Unable to delete key {}.".format(key) return ret @@ -595,7 +587,7 @@ def agent_maintenance(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Agent maintenance mode " "{0}ed.".format(kwargs["enable"]) + ret["message"] = "Agent maintenance mode " "{}ed.".format(kwargs["enable"]) else: ret["res"] = True ret["message"] = "Unable to change maintenance mode for agent." @@ -634,7 +626,7 @@ def agent_join(consul_url=None, token=None, address=None, **kwargs): if "wan" in kwargs: query_params["wan"] = kwargs["wan"] - function = "agent/join/{0}".format(address) + function = "agent/join/{}".format(address) res = _query( consul_url=consul_url, function=function, @@ -679,7 +671,7 @@ def agent_leave(consul_url=None, token=None, node=None): if not node: raise SaltInvocationError('Required argument "node" is missing.') - function = "agent/force-leave/{0}".format(node) + function = "agent/force-leave/{}".format(node) res = _query( consul_url=consul_url, function=function, @@ -689,10 +681,10 @@ def agent_leave(consul_url=None, token=None, node=None): ) if res["res"]: ret["res"] = True - ret["message"] = "Node {0} put in leave state.".format(node) + ret["message"] = "Node {} put in leave state.".format(node) else: ret["res"] = False - ret["message"] = "Unable to change state for {0}.".format(node) + ret["message"] = "Unable to change state for {}.".format(node) return ret @@ -775,7 +767,7 @@ def agent_check_register(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Check {0} added to agent.".format(kwargs["name"]) + ret["message"] = "Check {} added to agent.".format(kwargs["name"]) else: ret["res"] = False ret["message"] = "Unable to add check to agent." @@ -809,11 +801,11 @@ def agent_check_deregister(consul_url=None, token=None, checkid=None): if not checkid: raise SaltInvocationError('Required argument "checkid" is missing.') - function = "agent/check/deregister/{0}".format(checkid) + function = "agent/check/deregister/{}".format(checkid) res = _query(consul_url=consul_url, function=function, token=token, method="GET") if res["res"]: ret["res"] = True - ret["message"] = "Check {0} removed from agent.".format(checkid) + ret["message"] = "Check {} removed from agent.".format(checkid) else: ret["res"] = False ret["message"] = "Unable to remove check from agent." @@ -854,7 +846,7 @@ def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/pass/{0}".format(checkid) + function = "agent/check/pass/{}".format(checkid) res = _query( consul_url=consul_url, function=function, @@ -864,10 +856,10 @@ def agent_check_pass(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {0} marked as passing.".format(checkid) + ret["message"] = "Check {} marked as passing.".format(checkid) else: ret["res"] = False - ret["message"] = "Unable to update check {0}.".format(checkid) + ret["message"] = "Unable to update check {}.".format(checkid) return ret @@ -905,7 +897,7 @@ def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/warn/{0}".format(checkid) + function = "agent/check/warn/{}".format(checkid) res = _query( consul_url=consul_url, function=function, @@ -915,10 +907,10 @@ def agent_check_warn(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {0} marked as warning.".format(checkid) + ret["message"] = "Check {} marked as warning.".format(checkid) else: ret["res"] = False - ret["message"] = "Unable to update check {0}.".format(checkid) + ret["message"] = "Unable to update check {}.".format(checkid) return ret @@ -956,7 +948,7 @@ def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): if "note" in kwargs: query_params["note"] = kwargs["note"] - function = "agent/check/fail/{0}".format(checkid) + function = "agent/check/fail/{}".format(checkid) res = _query( consul_url=consul_url, function=function, @@ -966,10 +958,10 @@ def agent_check_fail(consul_url=None, token=None, checkid=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Check {0} marked as critical.".format(checkid) + ret["message"] = "Check {} marked as critical.".format(checkid) else: ret["res"] = False - ret["message"] = "Unable to update check {0}.".format(checkid) + ret["message"] = "Unable to update check {}.".format(checkid) return ret @@ -1016,7 +1008,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): return ret lc_kwargs = dict() - for k, v in six.iteritems(kwargs): + for k, v in kwargs.items(): lc_kwargs[k.lower()] = v if "name" in lc_kwargs: @@ -1044,7 +1036,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): if "check" in lc_kwargs: dd = dict() - for k, v in six.iteritems(lc_kwargs["check"]): + for k, v in lc_kwargs["check"].items(): dd[k.lower()] = v interval_required = False check_dd = dict() @@ -1070,6 +1062,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): del check_dd["Interval"] # not required, so ignore it if check_dd > 0: + data["Check"] = check_dd # if empty, ignore it function = "agent/service/register" @@ -1078,10 +1071,10 @@ def agent_service_register(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Service {0} registered on agent.".format(kwargs["name"]) + ret["message"] = "Service {} registered on agent.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Unable to register service {0}.".format(kwargs["name"]) + ret["message"] = "Unable to register service {}.".format(kwargs["name"]) return ret @@ -1113,16 +1106,16 @@ def agent_service_deregister(consul_url=None, token=None, serviceid=None): if not serviceid: raise SaltInvocationError('Required argument "serviceid" is missing.') - function = "agent/service/deregister/{0}".format(serviceid) + function = "agent/service/deregister/{}".format(serviceid) res = _query( consul_url=consul_url, function=function, token=token, method="PUT", data=data ) if res["res"]: ret["res"] = True - ret["message"] = "Service {0} removed from agent.".format(serviceid) + ret["message"] = "Service {} removed from agent.".format(serviceid) else: ret["res"] = False - ret["message"] = "Unable to remove service {0}.".format(serviceid) + ret["message"] = "Unable to remove service {}.".format(serviceid) return ret @@ -1167,17 +1160,17 @@ def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwa if "reason" in kwargs: query_params["reason"] = kwargs["reason"] - function = "agent/service/maintenance/{0}".format(serviceid) + function = "agent/service/maintenance/{}".format(serviceid) res = _query( consul_url=consul_url, token=token, function=function, query_params=query_params ) if res["res"]: ret["res"] = True - ret["message"] = "Service {0} set in " "maintenance mode.".format(serviceid) + ret["message"] = "Service {} set in " "maintenance mode.".format(serviceid) else: ret["res"] = False - ret["message"] = "Unable to set service " "{0} to maintenance mode.".format( + ret["message"] = "Unable to set service " "{} to maintenance mode.".format( serviceid ) return ret @@ -1247,14 +1240,14 @@ def session_create(consul_url=None, token=None, **kwargs): if "ttl" in kwargs: _ttl = kwargs["ttl"] - if six.text_type(_ttl).endswith("s"): + if str(_ttl).endswith("s"): _ttl = _ttl[:-1] if int(_ttl) < 0 or int(_ttl) > 3600: ret["message"] = ("TTL must be ", "between 0 and 3600.") ret["res"] = False return ret - data["TTL"] = "{0}s".format(_ttl) + data["TTL"] = "{}s".format(_ttl) function = "session/create" res = _query( @@ -1263,10 +1256,10 @@ def session_create(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Created session {0}.".format(kwargs["name"]) + ret["message"] = "Created session {}.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Unable to create session {0}.".format(kwargs["name"]) + ret["message"] = "Unable to create session {}.".format(kwargs["name"]) return ret @@ -1350,16 +1343,16 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "session/destroy/{0}".format(session) + function = "session/destroy/{}".format(session) res = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) if res["res"]: ret["res"] = True - ret["message"] = "Created Service {0}.".format(kwargs["name"]) + ret["message"] = "Created Service {}.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Unable to create service {0}.".format(kwargs["name"]) + ret["message"] = "Unable to create service {}.".format(kwargs["name"]) return ret @@ -1397,7 +1390,7 @@ def session_info(consul_url=None, token=None, session=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "session/info/{0}".format(session) + function = "session/info/{}".format(session) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1560,14 +1553,12 @@ def catalog_register(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Catalog registration " "for {0} successful.".format( + ret["message"] = "Catalog registration " "for {} successful.".format( kwargs["node"] ) else: ret["res"] = False - ret["message"] = "Catalog registration " "for {0} failed.".format( - kwargs["node"] - ) + ret["message"] = "Catalog registration " "for {} failed.".format(kwargs["node"]) ret["data"] = data return ret @@ -1624,10 +1615,10 @@ def catalog_deregister(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Catalog item {0} removed.".format(kwargs["node"]) + ret["message"] = "Catalog item {} removed.".format(kwargs["node"]) else: ret["res"] = False - ret["message"] = "Removing Catalog " "item {0} failed.".format(kwargs["node"]) + ret["message"] = "Removing Catalog " "item {} failed.".format(kwargs["node"]) return ret @@ -1767,7 +1758,7 @@ def catalog_service(consul_url=None, token=None, service=None, **kwargs): if "tag" in kwargs: query_params["tag"] = kwargs["tag"] - function = "catalog/service/{0}".format(service) + function = "catalog/service/{}".format(service) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1807,7 +1798,7 @@ def catalog_node(consul_url=None, token=None, node=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "catalog/node/{0}".format(node) + function = "catalog/node/{}".format(node) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1847,7 +1838,7 @@ def health_node(consul_url=None, token=None, node=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "health/node/{0}".format(node) + function = "health/node/{}".format(node) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1887,7 +1878,7 @@ def health_checks(consul_url=None, token=None, service=None, **kwargs): if "dc" in kwargs: query_params["dc"] = kwargs["dc"] - function = "health/checks/{0}".format(service) + function = "health/checks/{}".format(service) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1938,7 +1929,7 @@ def health_service(consul_url=None, token=None, service=None, **kwargs): if "passing" in kwargs: query_params["passing"] = kwargs["passing"] - function = "health/service/{0}".format(service) + function = "health/service/{}".format(service) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -1988,7 +1979,7 @@ def health_state(consul_url=None, token=None, state=None, **kwargs): ret["res"] = False return ret - function = "health/state/{0}".format(state) + function = "health/state/{}".format(state) ret = _query( consul_url=consul_url, function=function, token=token, query_params=query_params ) @@ -2100,10 +2091,10 @@ def acl_create(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "ACL {0} created.".format(kwargs["name"]) + ret["message"] = "ACL {} created.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Removing Catalog " "item {0} failed.".format(kwargs["name"]) + ret["message"] = "Removing Catalog " "item {} failed.".format(kwargs["name"]) return ret @@ -2163,10 +2154,10 @@ def acl_update(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "ACL {0} created.".format(kwargs["name"]) + ret["message"] = "ACL {} created.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Adding ACL " "{0} failed.".format(kwargs["name"]) + ret["message"] = "Adding ACL " "{} failed.".format(kwargs["name"]) return ret @@ -2201,17 +2192,17 @@ def acl_delete(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - function = "acl/delete/{0}".format(kwargs["id"]) + function = "acl/delete/{}".format(kwargs["id"]) res = _query( consul_url=consul_url, token=token, data=data, method="PUT", function=function ) if res["res"]: ret["res"] = True - ret["message"] = "ACL {0} deleted.".format(kwargs["id"]) + ret["message"] = "ACL {} deleted.".format(kwargs["id"]) else: ret["res"] = False - ret["message"] = "Removing ACL " "{0} failed.".format(kwargs["id"]) + ret["message"] = "Removing ACL " "{} failed.".format(kwargs["id"]) return ret @@ -2246,7 +2237,7 @@ def acl_info(consul_url=None, **kwargs): ret["res"] = False return ret - function = "acl/info/{0}".format(kwargs["id"]) + function = "acl/info/{}".format(kwargs["id"]) ret = _query(consul_url=consul_url, data=data, method="GET", function=function) return ret @@ -2282,17 +2273,17 @@ def acl_clone(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - function = "acl/clone/{0}".format(kwargs["id"]) + function = "acl/clone/{}".format(kwargs["id"]) res = _query( consul_url=consul_url, token=token, data=data, method="PUT", function=function ) if res["res"]: ret["res"] = True - ret["message"] = "ACL {0} cloned.".format(kwargs["name"]) + ret["message"] = "ACL {} cloned.".format(kwargs["name"]) ret["ID"] = ret["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL" "item {0} failed.".format(kwargs["name"]) + ret["message"] = "Cloning ACL" "item {} failed.".format(kwargs["name"]) return ret @@ -2377,7 +2368,7 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): if "tag" in kwargs: query_params = kwargs["tag"] - function = "event/fire/{0}".format(name) + function = "event/fire/{}".format(name) res = _query( consul_url=consul_url, token=token, @@ -2388,11 +2379,11 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): if res["res"]: ret["res"] = True - ret["message"] = "Event {0} fired.".format(name) + ret["message"] = "Event {} fired.".format(name) ret["data"] = ret["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL" "item {0} failed.".format(kwargs["name"]) + ret["message"] = "Cloning ACL" "item {} failed.".format(kwargs["name"]) return ret From f026031e7ac8176b972979c94ebc71c19db5d1d0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 30 Oct 2020 15:12:12 +0000 Subject: [PATCH 659/769] Fix incompatible types check --- src/saltext/consul/modules/consul.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index ac1c7f6..ce96957 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -1061,8 +1061,7 @@ def agent_service_register(consul_url=None, token=None, **kwargs): if "Interval" in check_dd: del check_dd["Interval"] # not required, so ignore it - if check_dd > 0: - + if check_dd: data["Check"] = check_dd # if empty, ignore it function = "agent/service/register" From b8833fac96cf5ca2f666f40da2e748121457d375 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 25 Nov 2020 14:56:33 +0000 Subject: [PATCH 660/769] Fix the `salt_ssh_cli` fixture --- tests/pytests/integration/conftest.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index b770c7c..6b9e569 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -4,6 +4,7 @@ PyTest fixtures """ + import pytest @@ -99,10 +100,16 @@ def salt_run_cli(salt_master): return salt_master.get_salt_run_cli() -@pytest.fixture(scope="package") -def salt_ssh_cli(salt_master): +@pytest.fixture(scope="module") +def salt_ssh_cli(salt_master, salt_ssh_roster_file, sshd_config_dir): """ The ``salt-ssh`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_ssh_cli() + return salt_master.get_salt_ssh_cli( + default_timeout=180, + roster_file=salt_ssh_roster_file, + target_host="localhost", + client_key=str(sshd_config_dir / "client_key"), + base_script_args=["--ignore-host-keys"], + ) From c86fc133a830073879324b8ac6d4c6c8774eab59 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 15 Jan 2021 07:35:38 +0000 Subject: [PATCH 661/769] Functional tests now use the module loaded from salt's loader --- tests/pytests/functional/conftest.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/pytests/functional/conftest.py diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py new file mode 100644 index 0000000..2948a7b --- /dev/null +++ b/tests/pytests/functional/conftest.py @@ -0,0 +1,88 @@ +import logging +import shutil + +import pytest +import salt.loader + +log = logging.getLogger(__name__) + + +class Loaders: + def __init__(self, opts): + self.opts = opts + self.context = {} + self._reset_state_funcs = [self.context.clear] + # Sadly, we can't use cached_property until Py3.6, and this is using a backports package + self._grains = self._utils = self._modules = None + self.opts["grains"] = self.grains + + def reset_state(self): + for func in self._reset_state_funcs: + func() + + @property + def grains(self): + if self._grains is None: + self._grains = salt.loader.grains(self.opts, context=self.context) + return self._grains + + @property + def utils(self): + if self._utils is None: + self._utils = salt.loader.utils(self.opts, context=self.context) + return self._utils + + @property + def modules(self): + if self._modules is None: + self._modules = salt.loader.minion_mods( + self.opts, context=self.context, utils=self.utils, initial_load=True + ) + return self._modules + + +@pytest.fixture(scope="module") +def state_tree_base(tmp_path_factory): + state_tree_path = tmp_path_factory.mktemp("state-tree-base") + try: + yield state_tree_path + finally: + shutil.rmtree(str(state_tree_path), ignore_errors=True) + + +@pytest.fixture(scope="module") +def state_tree(tmp_path_factory): + state_tree_path = tmp_path_factory.mktemp("state-tree-overrides") + try: + yield state_tree_path + finally: + shutil.rmtree(str(state_tree_path), ignore_errors=True) + + +@pytest.fixture(scope="module") +def minion_opts(salt_factories, state_tree_base, state_tree): + config_overrides = { + "file_client": "local", + "file_roots": {"base": [str(state_tree), str(state_tree_base)]}, + } + factory = salt_factories.get_salt_minion_daemon( + "functional-tests-minion", config_overrides=config_overrides, + ) + opts = factory.config.copy() + # opts["grains"] = + return opts + + +@pytest.fixture(scope="module") +def loaders(minion_opts): + return Loaders(minion_opts) + + +@pytest.fixture(autouse=True) +def reset_loaders_state(loaders): + try: + # Run the tests + yield + finally: + # Reset the loaders state + loaders.reset_state() From bfdffa667d210bd55606401bc39b66677d2aba8c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 19 Jan 2021 06:48:29 +0000 Subject: [PATCH 662/769] Migrate `integration.modules.test_state` to PyTest --- tests/pytests/functional/conftest.py | 47 +++++++++++++++++++++------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 2948a7b..dc6f460 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -2,7 +2,9 @@ import shutil import pytest +import salt.features import salt.loader +import salt.pillar log = logging.getLogger(__name__) @@ -12,9 +14,10 @@ def __init__(self, opts): self.opts = opts self.context = {} self._reset_state_funcs = [self.context.clear] - # Sadly, we can't use cached_property until Py3.6, and this is using a backports package - self._grains = self._utils = self._modules = None + self._grains = self._utils = self._modules = self._pillar = None self.opts["grains"] = self.grains + self.refresh_pillar() + salt.features.setup_features(self.opts) def reset_state(self): for func in self._reset_state_funcs: @@ -40,9 +43,30 @@ def modules(self): ) return self._modules + @property + def pillar(self): + if self._pillar is None: + self._pillar = salt.pillar.get_pillar( + self.opts, + self.opts["grains"], + self.opts["id"], + saltenv=self.opts["saltenv"], + pillarenv=self.opts.get("pillarenv"), + ).compile_pillar() + return self._pillar + + def refresh_pillar(self): + self._pillar = None + self.opts["pillar"] = self.pillar + + +@pytest.fixture(scope="package") +def minion_id(): + return "func-tests-minion" + @pytest.fixture(scope="module") -def state_tree_base(tmp_path_factory): +def state_tree(tmp_path_factory): state_tree_path = tmp_path_factory.mktemp("state-tree-base") try: yield state_tree_path @@ -51,8 +75,8 @@ def state_tree_base(tmp_path_factory): @pytest.fixture(scope="module") -def state_tree(tmp_path_factory): - state_tree_path = tmp_path_factory.mktemp("state-tree-overrides") +def state_tree_prod(tmp_path_factory): + state_tree_path = tmp_path_factory.mktemp("state-tree-prod") try: yield state_tree_path finally: @@ -60,17 +84,18 @@ def state_tree(tmp_path_factory): @pytest.fixture(scope="module") -def minion_opts(salt_factories, state_tree_base, state_tree): +def minion_opts( + salt_factories, minion_id, state_tree, state_tree_prod, +): config_overrides = { "file_client": "local", - "file_roots": {"base": [str(state_tree), str(state_tree_base)]}, + "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, + "features": {"enable_slsvars_fixes": True}, } factory = salt_factories.get_salt_minion_daemon( - "functional-tests-minion", config_overrides=config_overrides, + minion_id, config_overrides=config_overrides, ) - opts = factory.config.copy() - # opts["grains"] = - return opts + return factory.config.copy() @pytest.fixture(scope="module") From b1fcb55509e9461bc3b30d25e3bd9cdc6d520418 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 6 Jan 2021 09:25:09 -0800 Subject: [PATCH 663/769] moving test to tests/pytests/integration/modules/test_pillar.py --- src/saltext/consul/pillar/__init__.py | 48 +++++++++++++-------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 57f4b4d..46204c8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -893,7 +893,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): ) else: errors.append(msg) - mods.add(sls) + mods[sls] = state nstate = None if state: if not isinstance(state, dict): @@ -950,29 +950,27 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): nstate, mods, err = self.render_pstate( m_sub_sls, saltenv, mods, defaults ) - if nstate: - if key: - # If key is x:y, convert it to {x: {y: nstate}} - for key_fragment in reversed( - key.split(":") - ): - nstate = {key_fragment: nstate} - if not self.opts.get( - "pillar_includes_override_sls", False - ): - include_states.append(nstate) - else: - state = merge( - state, - nstate, - self.merge_strategy, - self.opts.get("renderer", "yaml"), - self.opts.get( - "pillar_merge_lists", False - ), - ) - if err: - errors += err + else: + nstate = mods[m_sub_sls] + if nstate: + if key: + # If key is x:y, convert it to {x: {y: nstate}} + for key_fragment in reversed(key.split(":")): + nstate = {key_fragment: nstate} + if not self.opts.get( + "pillar_includes_override_sls", False + ): + include_states.append(nstate) + else: + state = merge( + state, + nstate, + self.merge_strategy, + self.opts.get("renderer", "yaml"), + self.opts.get("pillar_merge_lists", False), + ) + if err: + errors += err if not self.opts.get("pillar_includes_override_sls", False): # merge included state(s) with the current state # merged last to ensure that its values are @@ -1002,7 +1000,7 @@ def render_pillar(self, matches, errors=None): errors = [] for saltenv, pstates in matches.items(): pstatefiles = [] - mods = set() + mods = {} for sls_match in pstates: matched_pstates = [] try: From b3371373258ec6aaf263ab757fc7dfe4f053f961 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 15 Feb 2021 10:49:49 +0000 Subject: [PATCH 664/769] Add support for functional testing of state modules --- tests/pytests/functional/conftest.py | 131 +++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index dc6f460..e6b969e 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,6 +1,7 @@ import logging import shutil +import attr import pytest import salt.features import salt.loader @@ -15,6 +16,7 @@ def __init__(self, opts): self.context = {} self._reset_state_funcs = [self.context.clear] self._grains = self._utils = self._modules = self._pillar = None + self._serializers = self._states = None self.opts["grains"] = self.grains self.refresh_pillar() salt.features.setup_features(self.opts) @@ -43,6 +45,49 @@ def modules(self): ) return self._modules + @property + def serializers(self): + if self._serializers is None: + self._serializers = salt.loader.serializers(self.opts) + return self._serializers + + @property + def states(self): + if self._states is None: + _states = salt.loader.states( + self.opts, + functions=self.modules, + utils=self.utils, + serializers=self.serializers, + context=self.context, + ) + # For state execution modules, because we'd have to almost copy/paste what salt.modules.state.single + # does, we actually "proxy" the call through salt.modules.state.single instead of calling the state + # execution modules directly. This was also how the non pytest test suite worked + # Let's load all modules now + _states._load_all() + # for funcname in list(_states): + # _states[funcname] + + # Now, we proxy loaded modules through salt.modules.state.single + for module_name in list(_states.loaded_modules): + for func_name in list(_states.loaded_modules[module_name]): + full_func_name = "{}.{}".format(module_name, func_name) + replacement_function = StateFunction( + self.modules.state.single, full_func_name + ) + _states._dict[full_func_name] = replacement_function + _states.loaded_modules[module_name][ + func_name + ] = replacement_function + setattr( + _states.loaded_modules[module_name], + func_name, + replacement_function, + ) + self._states = _states + return self._states + @property def pillar(self): if self._pillar is None: @@ -60,6 +105,92 @@ def refresh_pillar(self): self.opts["pillar"] = self.pillar +@attr.s +class StateResult: + raw = attr.ib() + state_id = attr.ib(init=False) + full_return = attr.ib(init=False) + filtered = attr.ib(init=False) + + @state_id.default + def _state_id(self): + return next(iter(self.raw.keys())) + + @full_return.default + def _full_return(self): + return self.raw[self.state_id] + + @filtered.default + def _filtered_default(self): + _filtered = {} + for key, value in self.full_return.items(): + if key.startswith("_") or key in ("duration", "start_time"): + continue + _filtered[key] = value + return _filtered + + @property + def name(self): + return self.full_return["name"] + + @property + def result(self): + return self.full_return["result"] + + @property + def changes(self): + return self.full_return["changes"] + + @property + def comment(self): + return self.full_return["comment"] + + def __eq__(self, value): + raise RuntimeError( + "Please assert comparissons with {}.filtered instead".format( + self.__class__.__name__ + ) + ) + + def __contains__(self, value): + raise RuntimeError( + "Please assert comparissons with {}.filtered instead".format( + self.__class__.__name__ + ) + ) + + def __bool__(self): + raise RuntimeError( + "Please assert comparissons with {}.filtered instead".format( + self.__class__.__name__ + ) + ) + + +@attr.s +class StateFunction: + proxy_func = attr.ib(repr=False) + state_func = attr.ib() + + def __call__(self, *args, **kwargs): + name = None + if args and len(args) == 1: + name = args[0] + if name is not None and "name" in kwargs: + raise RuntimeError( + "Either pass 'name' as the single argument to the call or remove 'name' as a keyword argument" + ) + if name is None: + name = kwargs.pop("name", None) + if name is None: + raise RuntimeError( + "'name' was not passed as the single argument to the function nor as a keyword argument" + ) + log.info("Calling state.single(%s, name=%s, %s)", self.state_func, name, kwargs) + result = self.proxy_func(self.state_func, name=name, **kwargs) + return StateResult(result) + + @pytest.fixture(scope="package") def minion_id(): return "func-tests-minion" From f4be38528bf2fe64a8a1d5d211464d9b14a4f869 Mon Sep 17 00:00:00 2001 From: Elias Probst Date: Sun, 29 Nov 2020 20:48:37 +0000 Subject: [PATCH 665/769] doc: remove `module` suffix from descriptions Until now, the listings of all kinds of modules looked quite inconsistent like this: - salt.states.smtp - salt.states.snapper module - salt.states.solrcloud module - salt.states.splunk The `module` suffix is redundant anyways, so let's remove it and do some further cleanup (adjusting overline/underline usage, underline length) using the following bash script: ```bash sed -i 's|\s\+module$||g' doc/ref/*/all/*.*.rst for rst in doc/ref/*/all/*.*.rst; do # enforce consistent overline/underline usage, remove all overlines if [[ $(head -n1 "${rst}") =~ ^=+$ ]]; then sed -i -e 1d "${rst}" fi # adjust the length of the underline to the length of the module name module_name=$(head -n1 "${rst}") underline=$(for i in $(seq ${#module_name}); do echo -n "="; done) sed -i '2s|^=\+$|'"${underline}"'|g' "${rst}" done ``` --- docs/ref/modules/all/salt.modules.consul.rst | 1 - docs/ref/pillar/all/salt.pillar.consul_pillar.rst | 4 ++-- docs/ref/sdb/all/salt.sdb.consul.rst | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/ref/modules/all/salt.modules.consul.rst b/docs/ref/modules/all/salt.modules.consul.rst index 4567678..132ec97 100644 --- a/docs/ref/modules/all/salt.modules.consul.rst +++ b/docs/ref/modules/all/salt.modules.consul.rst @@ -1,4 +1,3 @@ -=================== salt.modules.consul =================== diff --git a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst index c27449b..29e95b5 100644 --- a/docs/ref/pillar/all/salt.pillar.consul_pillar.rst +++ b/docs/ref/pillar/all/salt.pillar.consul_pillar.rst @@ -1,5 +1,5 @@ -salt.pillar.consul_pillar module -================================ +salt.pillar.consul_pillar +========================= .. automodule:: salt.pillar.consul_pillar :members: diff --git a/docs/ref/sdb/all/salt.sdb.consul.rst b/docs/ref/sdb/all/salt.sdb.consul.rst index 5a47d2b..043a783 100644 --- a/docs/ref/sdb/all/salt.sdb.consul.rst +++ b/docs/ref/sdb/all/salt.sdb.consul.rst @@ -1,5 +1,5 @@ -salt.sdb.consul module -====================== +salt.sdb.consul +=============== .. automodule:: salt.sdb.consul :members: From 266b3a7dae1d931f4553110a53a737650aed0b2f Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 19 Feb 2021 12:02:54 -0800 Subject: [PATCH 666/769] Adding the ability to clear and show the pillar cache enabled when pillar_cache is True. (#59411) * Adding the ability to clear and show the pillar cache enabled when pillar_cache is True. * Updating show_pillar_cache and clear_pillar_cache to return current state of the pillar_cache for the minion or minions. Adding tests. * Adding changelog file * Adding missing newline in salt/utils/cache.py. * Docs cleanup. Removing unnecessary saltenv. Moving test to pytest. * requested changes. * Update pillar.py Removing hardcoded saltenv. Setting default for saltenv to base. * Adding versionadded tags. * rewriting tests to more accurately demonstration that the clear is being cleared. * fixing versionadded tags to have the right version. --- src/saltext/consul/pillar/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 46204c8..d19a2e8 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -431,6 +431,14 @@ def fetch_pillar(self): ) return fresh_pillar.compile_pillar() + def clear_pillar(self): + """ + Clear the cache + """ + self.cache.clear() + + return True + def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs log.debug( "Scanning pillar cache for information about minion %s and pillarenv %s", From 0bf26618dcace21c81f7f344f248d25924803a5f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 26 Jan 2021 11:04:36 +0000 Subject: [PATCH 667/769] Drop Py2 and six on salt/__init__.py --- src/saltext/consul/__init__.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 40d7a8e..156fc81 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- """ Salt package """ -from __future__ import absolute_import, print_function, unicode_literals import importlib import sys @@ -16,7 +14,7 @@ sys.stderr.flush() -class TornadoImporter(object): +class TornadoImporter: def find_module(self, module_name, package_path=None): if module_name.startswith("tornado"): return self @@ -104,13 +102,7 @@ def __define_global_system_encoding_variable__(): # On linux default to ascii as a last resort encoding = "ascii" - # We can't use six.moves.builtins because these builtins get deleted sooner - # than expected. See: - # https://github.com/saltstack/salt/issues/21036 - if sys.version_info[0] < 3: - import __builtin__ as builtins # pylint: disable=incompatible-py3-code - else: - import builtins # pylint: disable=import-error + import builtins # Define the detected encoding as a built-in variable for ease of use setattr(builtins, "__salt_system_encoding__", encoding) From 33409321c5a054743b75c0268d4c1b95407d11c1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 2 Mar 2021 17:29:34 -0700 Subject: [PATCH 668/769] Wean off IOLoop.run_sync --- src/saltext/consul/__init__.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 156fc81..d33dafc 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -14,14 +14,25 @@ sys.stderr.flush() +USE_VENDORED_TORNADO = True + + class TornadoImporter: def find_module(self, module_name, package_path=None): - if module_name.startswith("tornado"): - return self + if USE_VENDORED_TORNADO: + if module_name.startswith("tornado"): + return self + else: + if module_name.startswith("salt.ext.tornado"): + return self return None def load_module(self, name): - mod = importlib.import_module("salt.ext.{}".format(name)) + if USE_VENDORED_TORNADO: + mod = importlib.import_module("salt.ext.{}".format(name)) + else: + # Remove 'salt.ext.' from the module + mod = importlib.import_module(name[9:]) sys.modules[name] = mod return mod From bdbeeb1385438f5d193a11eccfc477f753c0a26a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 5 Mar 2021 08:51:33 +0000 Subject: [PATCH 669/769] All matcher `match` functions must now accept a `minion_id` keyword arg --- src/saltext/consul/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d33dafc..3f372da 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,7 +2,6 @@ Salt package """ - import importlib import sys import warnings @@ -50,14 +49,6 @@ def load_module(self, name): append=True, ) -# While we are supporting Python2.6, hide nested with-statements warnings -warnings.filterwarnings( - "ignore", - "With-statements now directly support multiple context managers", - DeprecationWarning, - append=True, -) - # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( "ignore", From c999d5f70659ddf18d8685a65daf2a8a89d909ba Mon Sep 17 00:00:00 2001 From: Daniel Wozniak Date: Wed, 24 Mar 2021 07:52:25 -0700 Subject: [PATCH 670/769] Merge 3003 changes forward to the master branch (#59879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Merge 3002.6 bugfix changes (#59822) * Pass `CI_RUN` as an environment variable to the test run. This allows us to know if we're running the test suite under a CI environment or not and adapt/adjust if needed * Migrate `unit.setup` to PyTest * Backport ae36b15 just for test_install.py * Only skip tests on CI runs * Always store git sha in _version.py during installation * Fix PEP440 compliance. The wheel metadata version 1.2 states that the package version MUST be PEP440 compliant. This means that instead of `3002.2-511-g033c53eccb`, the salt version string should look like `3002.2+511.g033c53eccb`, a post release of `3002.2` ahead by 511 commits with the git sha `033c53eccb` * Fix and migrate `tests/unit/test_version.py` to PyTest * Skip test if `easy_install` is not available * We also need to be PEP440 compliant when there's no git history * Allow extra_filerefs as sanitized kwargs for SSH client * Fix regression on cmd.run when passing tuples as cmd Co-authored-by: Alexander Graul * Add unit tests to ensure cmd.run accepts tuples * Add unit test to check for extra_filerefs on SSH opts * Add changelog file * Fix comment for test case * Fix unit test to avoid failing on Windows * Skip failing test on windows * Fix test to work on Windows * Add all ssh kwargs to sanitize_kwargs method * Run pre-commit * Fix pylint * Fix cmdmod loglevel and module_names tests * Fix pre-commit * Skip ssh tests if binary does not exist * Use setup_loader for cmdmod test * Prevent argument injection in restartcheck * Add changelog for restartcheck fix * docs_3002.6 * Add back tests removed in merge Co-authored-by: Pedro Algarvio Co-authored-by: Megan Wilhite Co-authored-by: Bryce Larson Co-authored-by: Pablo Suárez Hernández Co-authored-by: Alexander Graul Co-authored-by: Frode Gundersen * Remove glance state module in favor of glance_image * update wording in changelog * bump deprecation warning to Silicon. * Updating warnutil version to Phosphorous. * Update salt/modules/keystone.py Co-authored-by: Megan Wilhite * Check $HOMEBREW_PREFIX when linking against libcrypto When loading `libcrypto`, Salt checks for a Homebrew installation of `openssl` at Homebrew's default prefix of `/usr/local`. However, on Apple Silicon Macs, Homebrew's default installation prefix is `/opt/homebrew`. On all platforms, the prefix is configurable. If Salt doesn't find one of those `libcrypto`s, it will fall back on the un-versioned `/usr/lib/libcrypto.dylib`, which will cause the following crash: Application Specific Information: /usr/lib/libcrypto.dylib abort() called Invalid dylib load. Clients should not load the unversioned libcrypto dylib as it does not have a stable ABI. This commit checks $HOMEBREW_PREFIX instead of hard-coding `/usr/local`. * Add test case * Add changelog for 59808 * Add changelog entry * Make _find_libcrypto fail on Big Sur if it can't find a library Right now, if `_find_libcrypto` can't find any externally-managed versions of libcrypto, it will fall back on the pre-Catalina un-versioned system libcrypto. This does not exist on Big Sur and it would be better to raise an exception here rather than crashing later when trying to open it. * Update _find_libcrypto tests This commit simplifies the unit tests for _find_libcrypto by mocking out the host's filesystem and testing the common libcrypto installations (brew, ports, etc.) on Big Sur. It simplifies the tests for falling back on system versions of libcrypto on previous versions of macOS. * Fix description of test_find_libcrypto_with_system_before_catalina * Patch sys.platform for test_rsax931 tests * modules/match: add missing "minion_id" in Pillar example The documented Pillar example for `match.filter_by` lacks the `minion_id` parameter. Without it, the assignment won't work as expected. - fix documentation - add tests: - to prove the misbehavior of the documented example - to prove the proper behaviour when supplying `minion_id` - to ensure some misbehaviour observed with compound matchers doesn't occur * Fix for issue #59773 - When instantiating the loader grab values of grains and pillars if they are NamedLoaderContext instances. - The loader uses a copy of opts. - Impliment deepcopy on NamedLoaderContext instances. * Add changelog for #59773 * _get_initial_pillar function returns pillar * Fix linter issues * Clean up test * Bump deprecation release for neutron * Uncomment Sulfur release name * Removing the _ext_nodes deprecation warning and alias. * Adding changelog. * Renaming changelog file. * Update 59804.removed * Initial pass at fips_mode config option * Fix pre-commit * Fix tests and add changelog * update docs 3003 * update docs 3003 - newline * Fix warts in changelog Co-authored-by: Pedro Algarvio Co-authored-by: Megan Wilhite Co-authored-by: Bryce Larson Co-authored-by: Pablo Suárez Hernández Co-authored-by: Alexander Graul Co-authored-by: Frode Gundersen Co-authored-by: Gareth J. Greenaway Co-authored-by: Gareth J. Greenaway Co-authored-by: Hoa-Long Tam Co-authored-by: krionbsd Co-authored-by: Elias Probst Co-authored-by: Frode Gundersen --- tests/pytests/unit/pillar/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/pytests/unit/pillar/__init__.py diff --git a/tests/pytests/unit/pillar/__init__.py b/tests/pytests/unit/pillar/__init__.py new file mode 100644 index 0000000..e69de29 From dc59528ae08cd24749a5073948010c45b833d4b6 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 5 May 2021 16:21:56 +0100 Subject: [PATCH 671/769] Bump to pytest-salt-factories >= 0.903.x --- tests/pytests/functional/conftest.py | 4 +--- tests/pytests/integration/conftest.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index e6b969e..4c1b4aa 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -223,9 +223,7 @@ def minion_opts( "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, "features": {"enable_slsvars_fixes": True}, } - factory = salt_factories.get_salt_minion_daemon( - minion_id, config_overrides=config_overrides, - ) + factory = salt_factories.salt_minion_daemon(minion_id, overrides=config_overrides,) return factory.config.copy() diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 6b9e569..39615eb 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -25,7 +25,7 @@ def salt_minion(salt_master, salt_minion_factory): assert salt_master.is_running() with salt_minion_factory.started(): # Sync All - salt_call_cli = salt_minion_factory.get_salt_call_cli() + salt_call_cli = salt_minion_factory.salt_call_cli() ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) assert ret.exitcode == 0, ret yield salt_minion_factory @@ -39,7 +39,7 @@ def salt_sub_minion(salt_master, salt_sub_minion_factory): assert salt_master.is_running() with salt_sub_minion_factory.started(): # Sync All - salt_call_cli = salt_sub_minion_factory.get_salt_call_cli() + salt_call_cli = salt_sub_minion_factory.salt_call_cli() ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) assert ret.exitcode == 0, ret yield salt_sub_minion_factory @@ -70,7 +70,7 @@ def salt_call_cli(salt_minion): The ``salt-call`` CLI as a fixture against the running minion """ assert salt_minion.is_running() - return salt_minion.get_salt_call_cli() + return salt_minion.salt_call_cli() @pytest.fixture(scope="package") @@ -79,7 +79,7 @@ def salt_cp_cli(salt_master): The ``salt-cp`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_cp_cli() + return salt_master.salt_cp_cli() @pytest.fixture(scope="package") @@ -88,7 +88,7 @@ def salt_key_cli(salt_master): The ``salt-key`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_key_cli() + return salt_master.salt_key_cli() @pytest.fixture(scope="package") @@ -97,7 +97,7 @@ def salt_run_cli(salt_master): The ``salt-run`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_run_cli() + return salt_master.salt_run_cli() @pytest.fixture(scope="module") @@ -106,8 +106,8 @@ def salt_ssh_cli(salt_master, salt_ssh_roster_file, sshd_config_dir): The ``salt-ssh`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_ssh_cli( - default_timeout=180, + return salt_master.salt_ssh_cli( + timeout=180, roster_file=salt_ssh_roster_file, target_host="localhost", client_key=str(sshd_config_dir / "client_key"), From d2176df908b9eb901df209238028ac020a9be3fa Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 16 Jun 2021 21:32:19 +0100 Subject: [PATCH 672/769] Bump to pytest-salt-factories 0.906.x --- tests/pytests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 39615eb..86557b5 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -61,7 +61,7 @@ def salt_cli(salt_master): The ``salt`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.get_salt_cli() + return salt_master.salt_cli() @pytest.fixture(scope="package") From 2c0f29b7b355de523f68433809d5bbc4a429c881 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 2 Jun 2021 07:01:19 +0100 Subject: [PATCH 673/769] Functional test modules can now provide ``minion_config_{defaults,overrides}`` --- tests/pytests/functional/conftest.py | 51 ++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 4c1b4aa..ce25cf8 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -93,7 +93,7 @@ def pillar(self): if self._pillar is None: self._pillar = salt.pillar.get_pillar( self.opts, - self.opts["grains"], + self.grains, self.opts["id"], saltenv=self.opts["saltenv"], pillarenv=self.opts.get("pillarenv"), @@ -147,21 +147,21 @@ def comment(self): def __eq__(self, value): raise RuntimeError( - "Please assert comparissons with {}.filtered instead".format( + "Please assert comparisons with {}.filtered instead".format( self.__class__.__name__ ) ) def __contains__(self, value): raise RuntimeError( - "Please assert comparissons with {}.filtered instead".format( + "Please assert comparisons with {}.filtered instead".format( self.__class__.__name__ ) ) def __bool__(self): raise RuntimeError( - "Please assert comparissons with {}.filtered instead".format( + "Please assert comparisons with {}.filtered instead".format( self.__class__.__name__ ) ) @@ -214,16 +214,45 @@ def state_tree_prod(tmp_path_factory): shutil.rmtree(str(state_tree_path), ignore_errors=True) +@pytest.fixture(scope="module") +def minion_config_defaults(): + """ + Functional test modules can provide this fixture to tweak the default configuration dictionary + passed to the minion factory + """ + return {} + + +@pytest.fixture(scope="module") +def minion_config_overrides(): + """ + Functional test modules can provide this fixture to tweak the configuration overrides dictionary + passed to the minion factory + """ + return {} + + @pytest.fixture(scope="module") def minion_opts( - salt_factories, minion_id, state_tree, state_tree_prod, + salt_factories, + minion_id, + state_tree, + state_tree_prod, + minion_config_defaults, + minion_config_overrides, ): - config_overrides = { - "file_client": "local", - "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, - "features": {"enable_slsvars_fixes": True}, - } - factory = salt_factories.salt_minion_daemon(minion_id, overrides=config_overrides,) + minion_config_overrides.update( + { + "file_client": "local", + "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, + "features": {"enable_slsvars_fixes": True}, + } + ) + factory = salt_factories.salt_minion_daemon( + minion_id, + defaults=minion_config_defaults or None, + overrides=minion_config_overrides, + ) return factory.config.copy() From 4801e4a3dfa6085f123b89050a1bc1dd8d56d3ec Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 1 Apr 2021 16:29:01 -0700 Subject: [PATCH 674/769] Pyupgrade and drop six --- src/saltext/consul/cache/__init__.py | 33 +++++++++------------- src/saltext/consul/cache/consul.py | 20 ++++++------- src/saltext/consul/modules/__init__.py | 1 - src/saltext/consul/pillar/__init__.py | 1 - src/saltext/consul/pillar/consul_pillar.py | 10 ++----- src/saltext/consul/sdb/__init__.py | 1 - src/saltext/consul/sdb/consul.py | 2 -- src/saltext/consul/states/__init__.py | 1 - tests/pytests/unit/__init__.py | 1 - tests/pytests/unit/modules/__init__.py | 1 - tests/pytests/unit/states/__init__.py | 1 - 11 files changed, 26 insertions(+), 46 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index b7c6ca9..72581a2 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -1,21 +1,16 @@ -# -*- coding: utf-8 -*- """ Loader mechanism for caching data, with data expiration, etc. .. versionadded:: 2016.11.0 """ -# Import Python libs -from __future__ import absolute_import, print_function, unicode_literals import logging import time -# Import Salt libs import salt.config import salt.loader import salt.syspaths -from salt.ext import six from salt.payload import Serial from salt.utils.odict import OrderedDict @@ -35,7 +30,7 @@ def factory(opts, **kwargs): return cls(opts, **kwargs) -class Cache(object): +class Cache: """ Base caching object providing access to the modular cache subsystem. @@ -83,7 +78,7 @@ def __init__(self, opts, cachedir=None, **kwargs): def __lazy_init(self): self._modules = salt.loader.cache(self.opts, self.serial) - fun = "{0}.init_kwargs".format(self.driver) + fun = "{}.init_kwargs".format(self.driver) if fun in self.modules: self._kwargs = self.modules[fun](self._kwargs) else: @@ -154,7 +149,7 @@ def store(self, bank, key, data): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.store".format(self.driver) + fun = "{}.store".format(self.driver) return self.modules[fun](bank, key, data, **self._kwargs) def fetch(self, bank, key): @@ -178,7 +173,7 @@ def fetch(self, bank, key): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.fetch".format(self.driver) + fun = "{}.fetch".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) def updated(self, bank, key): @@ -202,7 +197,7 @@ def updated(self, bank, key): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.updated".format(self.driver) + fun = "{}.updated".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) def flush(self, bank, key=None): @@ -223,7 +218,7 @@ def flush(self, bank, key=None): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.flush".format(self.driver) + fun = "{}.flush".format(self.driver) return self.modules[fun](bank, key=key, **self._kwargs) def list(self, bank): @@ -242,7 +237,7 @@ def list(self, bank): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.list".format(self.driver) + fun = "{}.list".format(self.driver) return self.modules[fun](bank, **self._kwargs) def contains(self, bank, key=None): @@ -267,7 +262,7 @@ def contains(self, bank, key=None): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ - fun = "{0}.contains".format(self.driver) + fun = "{}.contains".format(self.driver) return self.modules[fun](bank, key, **self._kwargs) @@ -281,7 +276,7 @@ class MemCache(Cache): data = {} def __init__(self, opts, **kwargs): - super(MemCache, self).__init__(opts, **kwargs) + super().__init__(opts, **kwargs) self.expire = opts.get("memcache_expire_seconds", 10) self.max = opts.get("memcache_max_items", 1024) self.cleanup = opts.get("memcache_full_cleanup", False) @@ -294,7 +289,7 @@ def __init__(self, opts, **kwargs): @classmethod def __cleanup(cls, expire): now = time.time() - for storage in six.itervalues(cls.data): + for storage in cls.data.values(): for key, data in list(storage.items()): if data[0] + expire < now: del storage[key] @@ -302,7 +297,7 @@ def __cleanup(cls, expire): break def _get_storage_id(self): - fun = "{0}.storage_id".format(self.driver) + fun = "{}.storage_id".format(self.driver) if fun in self.modules: return self.modules[fun](self.kwargs) else: @@ -338,7 +333,7 @@ def fetch(self, bank, key): return record[1] # Have no value for the key or value is expired - data = super(MemCache, self).fetch(bank, key) + data = super().fetch(bank, key) if len(self.storage) >= self.max: if self.cleanup: MemCache.__cleanup(self.expire) @@ -349,7 +344,7 @@ def fetch(self, bank, key): def store(self, bank, key, data): self.storage.pop((bank, key), None) - super(MemCache, self).store(bank, key, data) + super().store(bank, key, data) if len(self.storage) >= self.max: if self.cleanup: MemCache.__cleanup(self.expire) @@ -359,4 +354,4 @@ def store(self, bank, key, data): def flush(self, bank, key=None): self.storage.pop((bank, key), None) - super(MemCache, self).flush(bank, key) + super().flush(bank, key) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 8e61102..6f07b30 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Minion data cache plugin for Consul key/value data store. @@ -46,7 +45,6 @@ .. _`python-consul documentation`: https://python-consul.readthedocs.io/en/latest/#consul """ -from __future__ import absolute_import, print_function, unicode_literals import logging @@ -106,13 +104,13 @@ def store(bank, key, data): """ Store a key value. """ - c_key = "{0}/{1}".format(bank, key) + c_key = "{}/{}".format(bank, key) try: c_data = __context__["serial"].dumps(data) api.kv.put(c_key, c_data) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - "There was an error writing the key, {0}: {1}".format(c_key, exc) + "There was an error writing the key, {}: {}".format(c_key, exc) ) @@ -120,7 +118,7 @@ def fetch(bank, key): """ Fetch a key value. """ - c_key = "{0}/{1}".format(bank, key) + c_key = "{}/{}".format(bank, key) try: _, value = api.kv.get(c_key) if value is None: @@ -128,7 +126,7 @@ def fetch(bank, key): return __context__["serial"].loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - "There was an error reading the key, {0}: {1}".format(c_key, exc) + "There was an error reading the key, {}: {}".format(c_key, exc) ) @@ -139,12 +137,12 @@ def flush(bank, key=None): if key is None: c_key = bank else: - c_key = "{0}/{1}".format(bank, key) + c_key = "{}/{}".format(bank, key) try: return api.kv.delete(c_key, recurse=key is None) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - "There was an error removing the key, {0}: {1}".format(c_key, exc) + "There was an error removing the key, {}: {}".format(c_key, exc) ) @@ -156,7 +154,7 @@ def list_(bank): _, keys = api.kv.get(bank + "/", keys=True, separator="/") except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - 'There was an error getting the key "{0}": {1}'.format(bank, exc) + 'There was an error getting the key "{}": {}'.format(bank, exc) ) if keys is None: keys = [] @@ -178,10 +176,10 @@ def contains(bank, key): return True # any key could be a branch and a leaf at the same time in Consul else: try: - c_key = "{0}/{1}".format(bank, key) + c_key = "{}/{}".format(bank, key) _, value = api.kv.get(c_key) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( - "There was an error getting the key, {0}: {1}".format(c_key, exc) + "There was an error getting the key, {}: {}".format(c_key, exc) ) return value is not None diff --git a/src/saltext/consul/modules/__init__.py b/src/saltext/consul/modules/__init__.py index cb0f27f..95433ea 100644 --- a/src/saltext/consul/modules/__init__.py +++ b/src/saltext/consul/modules/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Execution Module Directory """ diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d19a2e8..78ad444 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -24,7 +24,6 @@ import salt.utils.dictupdate import salt.utils.url from salt.exceptions import SaltClientError -from salt.ext import six from salt.template import compile_template # Even though dictupdate is imported, invoking salt.utils.dictupdate.merge here diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index fd68ee4..076095f 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Use Consul K/V as a Pillar source with values parsed as YAML @@ -135,9 +134,7 @@ - consul: my_consul_config expand_keys=false """ -from __future__ import absolute_import, print_function, unicode_literals -# Import python libs import logging import re @@ -146,7 +143,6 @@ from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge -# Import third party libs try: import consul @@ -186,7 +182,7 @@ def ext_pillar(minion_id, pillar, conf): # pylint: disable=W0613 if minion_id not in minions: return {} - root_re = re.compile("(? Date: Fri, 25 Jun 2021 23:48:00 -0700 Subject: [PATCH 675/769] Always use tornado's ioloop --- src/saltext/consul/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3f372da..bfb60d9 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -40,6 +40,15 @@ def load_module(self, name): sys.meta_path = [TornadoImporter()] + sys.meta_path +def install_ioloop(): + # Make sure we always use tornado's ioloop + import salt.ext.tornado.ioloop + + salt.ext.tornado.ioloop.IOLoop().install() + + +install_ioloop() + # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( "once", # Show once From f5ad6643abd3aea307018a9b27f659a62c71c69b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 28 Jun 2021 15:40:47 -0700 Subject: [PATCH 676/769] Revert "Always use tornado's ioloop" This reverts commit faa16e7b23d34141962163b3bf30189726ab44ad. --- src/saltext/consul/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index bfb60d9..3f372da 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -40,15 +40,6 @@ def load_module(self, name): sys.meta_path = [TornadoImporter()] + sys.meta_path -def install_ioloop(): - # Make sure we always use tornado's ioloop - import salt.ext.tornado.ioloop - - salt.ext.tornado.ioloop.IOLoop().install() - - -install_ioloop() - # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( "once", # Show once From 5b6a4ac7969c897b5d2c54412a929b73e49491b0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 18 Jul 2021 13:27:50 +0100 Subject: [PATCH 677/769] Add a ``warnings`` attribute to ``StateResult`` --- tests/pytests/functional/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index ce25cf8..b1104a1 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -145,6 +145,10 @@ def changes(self): def comment(self): return self.full_return["comment"] + @property + def warnings(self): + return self.full_return.get("warnings") or [] + def __eq__(self, value): raise RuntimeError( "Please assert comparisons with {}.filtered instead".format( From 4451cbe395aff115c6f7f594d4feaf07d8ee4830 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 29 Jul 2021 23:15:02 +0100 Subject: [PATCH 678/769] Work around https://github.com/pypa/pip/pull/9450 (#60625) * Also pin the ``pip`` upgrade to be ``<21.2`` * Work around https://github.com/pypa/pip/pull/9450 See https://github.com/pypa/pip/issues/10212 Co-authored-by: Megan Wilhite Co-authored-by: Gareth J. Greenaway --- src/saltext/consul/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 3f372da..01ed844 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -119,3 +119,6 @@ def __define_global_system_encoding_variable__(): # This is now garbage collectable del __define_global_system_encoding_variable__ + +# Import Salt's logging machinery +import salt._logging.impl # isort:skip pylint: disable=unused-import From 8783d1dd4a308cbd7173d91219de7f7707a2673e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Aug 2021 07:25:24 +0100 Subject: [PATCH 679/769] Upgrade to black 21.7b0 --- src/saltext/consul/pillar/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 78ad444..0f2dc1b 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -239,7 +239,8 @@ def compile_pillar(self): load["ext"] = self.ext try: ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( - load, dictkey="pillar", + load, + dictkey="pillar", ) except Exception: # pylint: disable=broad-except log.exception("Exception getting pillar:") @@ -326,7 +327,8 @@ def compile_pillar(self): if self.ext: load["ext"] = self.ext ret_pillar = self.channel.crypted_transfer_decode_dictentry( - load, dictkey="pillar", + load, + dictkey="pillar", ) if not isinstance(ret_pillar, dict): @@ -818,7 +820,9 @@ def top_matches(self, top, reload=False): continue for match, data in body.items(): if self.matchers["confirm_top.confirm_top"]( - match, data, self.opts.get("nodegroups", {}), + match, + data, + self.opts.get("nodegroups", {}), ): if saltenv not in matches: matches[saltenv] = env_matches = [] @@ -939,7 +943,10 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): else: include_parts = sls.split(".")[:-1] sub_sls = ".".join(include_parts + [sub_sls[1:]]) - matches = fnmatch.filter(self.avail[saltenv], sub_sls,) + matches = fnmatch.filter( + self.avail[saltenv], + sub_sls, + ) matched_pstates.extend(matches) except KeyError: errors.extend( @@ -1150,7 +1157,10 @@ def ext_pillar(self, pillar, errors=None): ext = self._external_pillar_data(pillar, val, key) except Exception as exc: # pylint: disable=broad-except errors.append( - "Failed to load ext_pillar {}: {}".format(key, exc.__str__(),) + "Failed to load ext_pillar {}: {}".format( + key, + exc.__str__(), + ) ) log.error( "Exception caught loading ext_pillar '%s':\n%s", From 6041f221e3ccb08115e47265283bf5a497957141 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Aug 2021 08:40:21 +0100 Subject: [PATCH 680/769] Fix strings --- src/saltext/consul/cache/consul.py | 3 +- src/saltext/consul/modules/consul.py | 35 +++++++++++----------- src/saltext/consul/pillar/__init__.py | 32 ++++++++++---------- src/saltext/consul/pillar/consul_pillar.py | 4 ++- tests/pytests/functional/conftest.py | 6 ++-- 5 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 6f07b30..a10012c 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -94,7 +94,8 @@ def __virtual__(): except AttributeError: return ( False, - "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed", + "Failed to invoke consul.Consul, please make sure you have python-consul >=" + " 0.2.0 installed", ) return __virtualname__ diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index ce96957..814bc47 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -263,8 +263,9 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): for _l2 in conflicting_args: if _l1 in kwargs and _l2 in kwargs and _l1 != _l2: raise SaltInvocationError( - "Using arguments `{}` and `{}`" - " together is invalid.".format(_l1, _l2) + "Using arguments `{}` and `{}` together is invalid.".format( + _l1, _l2 + ) ) query_params = {} @@ -279,21 +280,19 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): if "cas" in kwargs: if _current["res"]: if kwargs["cas"] == 0: - ret["message"] = "Key {} exists, index " "must be non-zero.".format(key) + ret["message"] = "Key {} exists, index must be non-zero.".format(key) ret["res"] = False return ret if kwargs["cas"] != _current["data"]["ModifyIndex"]: - ret["message"] = "Key {} exists, but indexes " "do not match.".format( - key - ) + ret["message"] = "Key {} exists, but indexes do not match.".format(key) ret["res"] = False return ret query_params["cas"] = kwargs["cas"] else: ret[ "message" - ] = "Key {} does not exists, " "CAS argument can not be used.".format(key) + ] = "Key {} does not exists, CAS argument can not be used.".format(key) ret["res"] = False return ret @@ -587,7 +586,7 @@ def agent_maintenance(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Agent maintenance mode " "{}ed.".format(kwargs["enable"]) + ret["message"] = "Agent maintenance mode {}ed.".format(kwargs["enable"]) else: ret["res"] = True ret["message"] = "Unable to change maintenance mode for agent." @@ -1166,10 +1165,10 @@ def agent_service_maintenance(consul_url=None, token=None, serviceid=None, **kwa if res["res"]: ret["res"] = True - ret["message"] = "Service {} set in " "maintenance mode.".format(serviceid) + ret["message"] = "Service {} set in maintenance mode.".format(serviceid) else: ret["res"] = False - ret["message"] = "Unable to set service " "{} to maintenance mode.".format( + ret["message"] = "Unable to set service {} to maintenance mode.".format( serviceid ) return ret @@ -1552,12 +1551,12 @@ def catalog_register(consul_url=None, token=None, **kwargs): ) if res["res"]: ret["res"] = True - ret["message"] = "Catalog registration " "for {} successful.".format( + ret["message"] = "Catalog registration for {} successful.".format( kwargs["node"] ) else: ret["res"] = False - ret["message"] = "Catalog registration " "for {} failed.".format(kwargs["node"]) + ret["message"] = "Catalog registration for {} failed.".format(kwargs["node"]) ret["data"] = data return ret @@ -1617,7 +1616,7 @@ def catalog_deregister(consul_url=None, token=None, **kwargs): ret["message"] = "Catalog item {} removed.".format(kwargs["node"]) else: ret["res"] = False - ret["message"] = "Removing Catalog " "item {} failed.".format(kwargs["node"]) + ret["message"] = "Removing Catalog item {} failed.".format(kwargs["node"]) return ret @@ -2093,7 +2092,7 @@ def acl_create(consul_url=None, token=None, **kwargs): ret["message"] = "ACL {} created.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Removing Catalog " "item {} failed.".format(kwargs["name"]) + ret["message"] = "Removing Catalog item {} failed.".format(kwargs["name"]) return ret @@ -2156,7 +2155,7 @@ def acl_update(consul_url=None, token=None, **kwargs): ret["message"] = "ACL {} created.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Adding ACL " "{} failed.".format(kwargs["name"]) + ret["message"] = "Adding ACL {} failed.".format(kwargs["name"]) return ret @@ -2201,7 +2200,7 @@ def acl_delete(consul_url=None, token=None, **kwargs): ret["message"] = "ACL {} deleted.".format(kwargs["id"]) else: ret["res"] = False - ret["message"] = "Removing ACL " "{} failed.".format(kwargs["id"]) + ret["message"] = "Removing ACL {} failed.".format(kwargs["id"]) return ret @@ -2282,7 +2281,7 @@ def acl_clone(consul_url=None, token=None, **kwargs): ret["ID"] = ret["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL" "item {} failed.".format(kwargs["name"]) + ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) return ret @@ -2382,7 +2381,7 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): ret["data"] = ret["data"] else: ret["res"] = False - ret["message"] = "Cloning ACL" "item {} failed.".format(kwargs["name"]) + ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) return ret diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0f2dc1b..a352561 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -167,7 +167,7 @@ def get_subconfig(opts_key): if "pass_to_ext_pillars" in opts: if not isinstance(opts["pass_to_ext_pillars"], list): log.exception("'pass_to_ext_pillars' config is malformed.") - raise SaltClientError("'pass_to_ext_pillars' config is " "malformed.") + raise SaltClientError("'pass_to_ext_pillars' config is malformed.") for key in opts["pass_to_ext_pillars"]: salt.utils.dictupdate.update( extra_data, @@ -247,9 +247,9 @@ def compile_pillar(self): raise SaltClientError("Exception getting pillar.") if not isinstance(ret_pillar, dict): - msg = ( - "Got a bad pillar from master, type {}, expecting dict: " "{}" - ).format(type(ret_pillar).__name__, ret_pillar) + msg = "Got a bad pillar from master, type {}, expecting dict: {}".format( + type(ret_pillar).__name__, ret_pillar + ) log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! raise SaltClientError(msg) @@ -629,7 +629,8 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): opts["pillar_roots"][env] = opts["pillar_roots"].pop("__env__") else: log.debug( - "pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", + "pillar_roots __env__ ignored (environment '%s' found in" + " pillar_roots)", env, ) opts["pillar_roots"].pop("__env__") @@ -724,9 +725,9 @@ def get_tops(self): ) except Exception as exc: # pylint: disable=broad-except errors.append( - ( - "Rendering Top file {} failed, render error" ":\n{}" - ).format(sls, exc) + "Rendering Top file {} failed, render error:\n{}".format( + sls, exc + ) ) done[saltenv].append(sls) for saltenv in pops: @@ -846,7 +847,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): if not fn_: if sls in self.ignored_pillars.get(saltenv, []): log.debug( - "Skipping ignored and missing SLS '%s' in " "environment '%s'", + "Skipping ignored and missing SLS '%s' in environment '%s'", sls, saltenv, ) @@ -854,14 +855,13 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): elif self.opts["pillar_roots"].get(saltenv): msg = ( "Specified SLS '{}' in environment '{}' is not" - " available on the salt master" - ).format(sls, saltenv) + " available on the salt master".format(sls, saltenv) + ) log.error(msg) errors.append(msg) else: - msg = ( - "Specified SLS '{}' in environment '{}' was not " - "found. ".format(sls, saltenv) + msg = "Specified SLS '{}' in environment '{}' was not found. ".format( + sls, saltenv ) if self.opts.get("__git_pillar", False) is True: msg += ( @@ -951,8 +951,8 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): except KeyError: errors.extend( [ - "No matching pillar environment for environment " - "'{}' found".format(saltenv) + "No matching pillar environment for environment" + " '{}' found".format(saltenv) ] ) matched_pstates = [sub_sls] diff --git a/src/saltext/consul/pillar/consul_pillar.py b/src/saltext/consul/pillar/consul_pillar.py index 076095f..2b28b29 100644 --- a/src/saltext/consul/pillar/consul_pillar.py +++ b/src/saltext/consul/pillar/consul_pillar.py @@ -147,7 +147,9 @@ import consul if not hasattr(consul, "__version__"): - consul.__version__ = "0.1" # Some packages has no version, and so this pillar crashes on access to it. + consul.__version__ = ( # Some packages has no version, and so this pillar crashes on access to it. + "0.1" + ) except ImportError: consul = None diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index b1104a1..43a8847 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -182,13 +182,15 @@ def __call__(self, *args, **kwargs): name = args[0] if name is not None and "name" in kwargs: raise RuntimeError( - "Either pass 'name' as the single argument to the call or remove 'name' as a keyword argument" + "Either pass 'name' as the single argument to the call or remove 'name'" + " as a keyword argument" ) if name is None: name = kwargs.pop("name", None) if name is None: raise RuntimeError( - "'name' was not passed as the single argument to the function nor as a keyword argument" + "'name' was not passed as the single argument to the function nor as a" + " keyword argument" ) log.info("Calling state.single(%s, name=%s, %s)", self.state_func, name, kwargs) result = self.proxy_func(self.state_func, name=name, **kwargs) From 60e94c7aaa026706b56e7ee5176bc28a9f190893 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 16 Aug 2021 18:03:57 +0100 Subject: [PATCH 681/769] Remove temporary ``StateReturnAsserts`` class (#60737) * Pin to ``pip>=20.2.4,<21.2`` on the created virtualenvs * Use the static requirements when installing salt into the virtualenv This should fix the freebsd failures * ``SaltVirtualEnv`` will always use ``USE_STATIC_REQUIREMENTS=1`` * Update to ``pytest-salt-factories==0.907.x`` * Remove the temporary ``StateReturnAsserts`` class. In the process, a few functions from the ``state`` module are now wrapped to allow a more pythonic assertion against it's returns through ``StateReturn``. We also introduce ``MultiStateReturn`` for full state runs. * Update tests that relied on ``StateReturnAsserts`` or it's pytest helper function --- tests/pytests/functional/conftest.py | 192 +-------------------------- 1 file changed, 1 insertion(+), 191 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 43a8847..1e81032 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,202 +1,12 @@ import logging import shutil -import attr import pytest -import salt.features -import salt.loader -import salt.pillar +from saltfactories.utils.functional import Loaders log = logging.getLogger(__name__) -class Loaders: - def __init__(self, opts): - self.opts = opts - self.context = {} - self._reset_state_funcs = [self.context.clear] - self._grains = self._utils = self._modules = self._pillar = None - self._serializers = self._states = None - self.opts["grains"] = self.grains - self.refresh_pillar() - salt.features.setup_features(self.opts) - - def reset_state(self): - for func in self._reset_state_funcs: - func() - - @property - def grains(self): - if self._grains is None: - self._grains = salt.loader.grains(self.opts, context=self.context) - return self._grains - - @property - def utils(self): - if self._utils is None: - self._utils = salt.loader.utils(self.opts, context=self.context) - return self._utils - - @property - def modules(self): - if self._modules is None: - self._modules = salt.loader.minion_mods( - self.opts, context=self.context, utils=self.utils, initial_load=True - ) - return self._modules - - @property - def serializers(self): - if self._serializers is None: - self._serializers = salt.loader.serializers(self.opts) - return self._serializers - - @property - def states(self): - if self._states is None: - _states = salt.loader.states( - self.opts, - functions=self.modules, - utils=self.utils, - serializers=self.serializers, - context=self.context, - ) - # For state execution modules, because we'd have to almost copy/paste what salt.modules.state.single - # does, we actually "proxy" the call through salt.modules.state.single instead of calling the state - # execution modules directly. This was also how the non pytest test suite worked - # Let's load all modules now - _states._load_all() - # for funcname in list(_states): - # _states[funcname] - - # Now, we proxy loaded modules through salt.modules.state.single - for module_name in list(_states.loaded_modules): - for func_name in list(_states.loaded_modules[module_name]): - full_func_name = "{}.{}".format(module_name, func_name) - replacement_function = StateFunction( - self.modules.state.single, full_func_name - ) - _states._dict[full_func_name] = replacement_function - _states.loaded_modules[module_name][ - func_name - ] = replacement_function - setattr( - _states.loaded_modules[module_name], - func_name, - replacement_function, - ) - self._states = _states - return self._states - - @property - def pillar(self): - if self._pillar is None: - self._pillar = salt.pillar.get_pillar( - self.opts, - self.grains, - self.opts["id"], - saltenv=self.opts["saltenv"], - pillarenv=self.opts.get("pillarenv"), - ).compile_pillar() - return self._pillar - - def refresh_pillar(self): - self._pillar = None - self.opts["pillar"] = self.pillar - - -@attr.s -class StateResult: - raw = attr.ib() - state_id = attr.ib(init=False) - full_return = attr.ib(init=False) - filtered = attr.ib(init=False) - - @state_id.default - def _state_id(self): - return next(iter(self.raw.keys())) - - @full_return.default - def _full_return(self): - return self.raw[self.state_id] - - @filtered.default - def _filtered_default(self): - _filtered = {} - for key, value in self.full_return.items(): - if key.startswith("_") or key in ("duration", "start_time"): - continue - _filtered[key] = value - return _filtered - - @property - def name(self): - return self.full_return["name"] - - @property - def result(self): - return self.full_return["result"] - - @property - def changes(self): - return self.full_return["changes"] - - @property - def comment(self): - return self.full_return["comment"] - - @property - def warnings(self): - return self.full_return.get("warnings") or [] - - def __eq__(self, value): - raise RuntimeError( - "Please assert comparisons with {}.filtered instead".format( - self.__class__.__name__ - ) - ) - - def __contains__(self, value): - raise RuntimeError( - "Please assert comparisons with {}.filtered instead".format( - self.__class__.__name__ - ) - ) - - def __bool__(self): - raise RuntimeError( - "Please assert comparisons with {}.filtered instead".format( - self.__class__.__name__ - ) - ) - - -@attr.s -class StateFunction: - proxy_func = attr.ib(repr=False) - state_func = attr.ib() - - def __call__(self, *args, **kwargs): - name = None - if args and len(args) == 1: - name = args[0] - if name is not None and "name" in kwargs: - raise RuntimeError( - "Either pass 'name' as the single argument to the call or remove 'name'" - " as a keyword argument" - ) - if name is None: - name = kwargs.pop("name", None) - if name is None: - raise RuntimeError( - "'name' was not passed as the single argument to the function nor as a" - " keyword argument" - ) - log.info("Calling state.single(%s, name=%s, %s)", self.state_func, name, kwargs) - result = self.proxy_func(self.state_func, name=name, **kwargs) - return StateResult(result) - - @pytest.fixture(scope="package") def minion_id(): return "func-tests-minion" From 9743a9781f55bfac4ac2597460889afcb8c91e80 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 30 Apr 2021 13:27:57 -0700 Subject: [PATCH 682/769] Initial work to add some basic tests for deltaproxy --- tests/pytests/integration/conftest.py | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 86557b5..34bea99 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -5,8 +5,12 @@ PyTest fixtures """ +import logging + import pytest +log = logging.getLogger(__name__) + @pytest.fixture(scope="package") def salt_master(salt_master_factory): @@ -55,6 +59,71 @@ def salt_proxy(salt_master, salt_proxy_factory): yield salt_proxy_factory +@pytest.fixture(scope="package") +def deltaproxy_pillar_tree(base_env_pillar_tree_root_dir, salt_delta_proxy_factory): + """ + Create the pillar files for controlproxy and two dummy proxy minions + """ + log.debug("==== running deltaproxy_pillar_tree ====") + top_file = """ + base: + '{}': + - controlproxy + dummy_proxy_one: + - dummy_proxy_one + dummy_proxy_two: + - dummy_proxy_two + """.format( + salt_delta_proxy_factory.id + ) + controlproxy_pillar_file = """ + proxy: + proxytype: deltaproxy + ids: + - dummy_proxy_one + - dummy_proxy_two + """ + + dummy_proxy_one_pillar_file = """ + proxy: + proxytype: dummy + """ + + dummy_proxy_two_pillar_file = """ + proxy: + proxytype: dummy + """ + + top_tempfile = pytest.helpers.temp_file( + "top.sls", top_file, base_env_pillar_tree_root_dir + ) + controlproxy_tempfile = pytest.helpers.temp_file( + "controlproxy.sls", controlproxy_pillar_file, base_env_pillar_tree_root_dir + ) + dummy_proxy_one_tempfile = pytest.helpers.temp_file( + "dummy_proxy_one.sls", + dummy_proxy_one_pillar_file, + base_env_pillar_tree_root_dir, + ) + dummy_proxy_two_tempfile = pytest.helpers.temp_file( + "dummy_proxy_two.sls", + dummy_proxy_two_pillar_file, + base_env_pillar_tree_root_dir, + ) + yield + + +@pytest.fixture(scope="package") +def salt_delta_proxy(salt_master, salt_delta_proxy_factory, deltaproxy_pillar_tree): + """ + A running salt-proxy fixture + """ + assert salt_master.is_running() + log.debug("==== running salt_delta_proxy ====") + with salt_delta_proxy_factory.started(): + yield salt_delta_proxy_factory + + @pytest.fixture(scope="package") def salt_cli(salt_master): """ From 25473ef10bba17d30a604b04e1a1a3c24a694c7c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 3 May 2021 12:54:31 -0700 Subject: [PATCH 683/769] make sure we set the right proxy ID in case the controlproxy ID is defined in the proxy configuration file. Updating tests. --- tests/pytests/integration/conftest.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 34bea99..09485bb 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -64,14 +64,13 @@ def deltaproxy_pillar_tree(base_env_pillar_tree_root_dir, salt_delta_proxy_facto """ Create the pillar files for controlproxy and two dummy proxy minions """ - log.debug("==== running deltaproxy_pillar_tree ====") top_file = """ base: '{}': - controlproxy - dummy_proxy_one: + dummy_proxy_one: - dummy_proxy_one - dummy_proxy_two: + dummy_proxy_two: - dummy_proxy_two """.format( salt_delta_proxy_factory.id @@ -110,7 +109,8 @@ def deltaproxy_pillar_tree(base_env_pillar_tree_root_dir, salt_delta_proxy_facto dummy_proxy_two_pillar_file, base_env_pillar_tree_root_dir, ) - yield + with top_tempfile, controlproxy_tempfile, dummy_proxy_one_tempfile, dummy_proxy_two_tempfile: + yield @pytest.fixture(scope="package") @@ -119,7 +119,6 @@ def salt_delta_proxy(salt_master, salt_delta_proxy_factory, deltaproxy_pillar_tr A running salt-proxy fixture """ assert salt_master.is_running() - log.debug("==== running salt_delta_proxy ====") with salt_delta_proxy_factory.started(): yield salt_delta_proxy_factory From 5d2bc8330659d0b6e3389d5d0090a14cc742127a Mon Sep 17 00:00:00 2001 From: mkirkland4874 <36466711+mkirkland4874@users.noreply.github.com> Date: Fri, 1 Jan 2021 18:57:40 +0000 Subject: [PATCH 684/769] Preserve order of pillar_roots in _get_envs() _get_envs() returns pillar_roots in a non-deterministic order. Update _get_envs() to preserve the order of pillar_roots similar to #e36d683 for file_roots in state.py. Fixes #24501 --- src/saltext/consul/pillar/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index a352561..0fbb3fe 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -640,9 +640,9 @@ def _get_envs(self): """ Pull the file server environments out of the master options """ - envs = {"base"} + envs = ["base"] if "pillar_roots" in self.opts: - envs.update(list(self.opts["pillar_roots"])) + envs.extend([x for x in list(self.opts["pillar_roots"]) if x not in envs]) return envs def get_tops(self): From 285130dde1bcd2e71e842c9f7383670c7d6a7d20 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 16 Sep 2021 18:37:43 +0100 Subject: [PATCH 685/769] Add regression tests for #24501 --- src/saltext/consul/pillar/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 0fbb3fe..22f5c3a 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -2,11 +2,9 @@ Render the pillar data """ - import collections import copy import fnmatch -import inspect import logging import os import sys @@ -629,8 +627,7 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): opts["pillar_roots"][env] = opts["pillar_roots"].pop("__env__") else: log.debug( - "pillar_roots __env__ ignored (environment '%s' found in" - " pillar_roots)", + "pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", env, ) opts["pillar_roots"].pop("__env__") From b0db9f16515ed8bec07b6a71ea2d0937c47da145 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 24 Sep 2021 19:48:04 -0700 Subject: [PATCH 686/769] Deprecate salt.payload.Serial --- src/saltext/consul/cache/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 72581a2..e6a7031 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -10,8 +10,8 @@ import salt.config import salt.loader +import salt.payload import salt.syspaths -from salt.payload import Serial from salt.utils.odict import OrderedDict log = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class Cache: Salt cache subsystem is organized as a tree with nodes and leafs like a filesystem. Cache consists of banks. Each bank can contain a number of keys. Each key can contain a dict or any other object serializable with - `salt.payload.Serial`. I.e. any data object in the cache can be + `salt.payload`. I.e. any data object in the cache can be addressed by the path to the bank and the key name: bank: 'minions/alpha' key: 'data' @@ -71,7 +71,7 @@ def __init__(self, opts, cachedir=None, **kwargs): else: self.cachedir = cachedir self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"]) - self.serial = Serial(opts) + self.serial = salt.payload self._modules = None self._kwargs = kwargs self._kwargs["cachedir"] = self.cachedir From df2a754089daf104fb052ea26e290ea6bedc01d2 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 24 Sep 2021 23:01:17 -0700 Subject: [PATCH 687/769] Remove references to self.serial --- src/saltext/consul/cache/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index e6a7031..118f77c 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -71,13 +71,12 @@ def __init__(self, opts, cachedir=None, **kwargs): else: self.cachedir = cachedir self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"]) - self.serial = salt.payload self._modules = None self._kwargs = kwargs self._kwargs["cachedir"] = self.cachedir def __lazy_init(self): - self._modules = salt.loader.cache(self.opts, self.serial) + self._modules = salt.loader.cache(self.opts, salt.payload) fun = "{}.init_kwargs".format(self.driver) if fun in self.modules: self._kwargs = self.modules[fun](self._kwargs) From 6717794d4c1ad3d921d591d43080808ed1a165bb Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 25 Sep 2021 00:39:34 -0700 Subject: [PATCH 688/769] Clean up more serial references --- src/saltext/consul/cache/__init__.py | 8 +------- src/saltext/consul/cache/consul.py | 5 +++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 118f77c..f72b93e 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -40,12 +40,6 @@ class Cache: The name of the cache driver to use. This is the name of the python module of the `salt.cache` package. Default is `localfs`. - :param serial: - The module of `salt.serializers` package that should be used by the cache - driver to store data. - If a driver can't use a specific module or uses specific objects storage - it can ignore this parameter. - Terminology. Salt cache subsystem is organized as a tree with nodes and leafs like a @@ -142,7 +136,7 @@ def store(self, bank, key, data): :param data: The data which will be stored in the cache. This data should be - in a format which can be serialized by msgpack/json/yaml/etc. + in a format which can be serialized by msgpack. :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index a10012c..fe5a266 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -48,6 +48,7 @@ import logging +import salt.payload from salt.exceptions import SaltCacheError try: @@ -107,7 +108,7 @@ def store(bank, key, data): """ c_key = "{}/{}".format(bank, key) try: - c_data = __context__["serial"].dumps(data) + c_data = salt.payload.dumps(data) api.kv.put(c_key, c_data) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( @@ -124,7 +125,7 @@ def fetch(bank, key): _, value = api.kv.get(c_key) if value is None: return {} - return __context__["serial"].loads(value["Value"]) + return salt.payload.loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( "There was an error reading the key, {}: {}".format(c_key, exc) From db152fd1107ec350ec2b8d548347ab0ab4266709 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 25 Sep 2021 00:44:56 -0700 Subject: [PATCH 689/769] Clean up salt.cache --- src/saltext/consul/cache/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index f72b93e..464493e 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -10,7 +10,6 @@ import salt.config import salt.loader -import salt.payload import salt.syspaths from salt.utils.odict import OrderedDict @@ -70,7 +69,7 @@ def __init__(self, opts, cachedir=None, **kwargs): self._kwargs["cachedir"] = self.cachedir def __lazy_init(self): - self._modules = salt.loader.cache(self.opts, salt.payload) + self._modules = salt.loader.cache(self.opts) fun = "{}.init_kwargs".format(self.driver) if fun in self.modules: self._kwargs = self.modules[fun](self._kwargs) From ce994a3fb6ca66c5761d13510b5132ff45daf867 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Wed, 29 Sep 2021 13:28:53 -0400 Subject: [PATCH 690/769] Refresh pillar cache when saltutil.refresh_pillar is run --- src/saltext/consul/pillar/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 22f5c3a..b1afb20 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -44,6 +44,7 @@ def get_pillar( pillar_override=None, pillarenv=None, extra_minion_data=None, + clean_cache=False, ): """ Return the correct pillar driver based on the file_client option @@ -78,6 +79,7 @@ def get_pillar( functions=funcs, pillar_override=pillar_override, pillarenv=pillarenv, + clean_cache=clean_cache, ) return ptype( opts, @@ -103,6 +105,7 @@ def get_async_pillar( pillar_override=None, pillarenv=None, extra_minion_data=None, + clean_cache=False, ): """ Return the correct pillar driver based on the file_client option @@ -115,6 +118,21 @@ def get_async_pillar( ptype = {"remote": AsyncRemotePillar, "local": AsyncPillar}.get( file_client, AsyncPillar ) + if file_client == "remote": + # AsyncPillar does not currently support calls to PillarCache + # clean_cache is a kwarg for PillarCache + return ptype( + opts, + grains, + minion_id, + saltenv, + ext, + functions=funcs, + pillar_override=pillar_override, + pillarenv=pillarenv, + extra_minion_data=extra_minion_data, + clean_cache=clean_cache, + ) return ptype( opts, grains, @@ -193,6 +211,7 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, + clean_cache=False, ): self.opts = opts self.opts["saltenv"] = saltenv @@ -217,6 +236,7 @@ def __init__( merge_lists=True, ) self._closing = False + self.clean_cache = clean_cache @salt.ext.tornado.gen.coroutine def compile_pillar(self): @@ -233,6 +253,8 @@ def compile_pillar(self): "ver": "2", "cmd": "_pillar", } + if self.clean_cache: + load["clean_cache"] = self.clean_cache if self.ext: load["ext"] = self.ext try: @@ -379,6 +401,7 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, + clean_cache=False, ): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -391,6 +414,7 @@ def __init__( self.functions = functions self.pillar_override = pillar_override self.pillarenv = pillarenv + self.clean_cache = clean_cache if saltenv is None: self.saltenv = "base" @@ -439,6 +463,8 @@ def clear_pillar(self): return True def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs + if self.clean_cache: + self.clear_pillar() log.debug( "Scanning pillar cache for information about minion %s and pillarenv %s", self.minion_id, From 02f9b77ce4ee405967a24381f5837e5804eecf2d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Sep 2021 16:39:37 +0100 Subject: [PATCH 691/769] Redirect imports of ``salt.ext.six`` to ``six`` Fixes #60966 --- src/saltext/consul/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 01ed844..0a18a9b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -36,8 +36,20 @@ def load_module(self, name): return mod +class SixRedirectImporter: + def find_module(self, module_name, package_path=None): + if module_name.startswith("salt.ext.six"): + return self + return None + + def load_module(self, name): + mod = importlib.import_module(name[9:]) + sys.modules[name] = mod + return mod + + # Try our importer first -sys.meta_path = [TornadoImporter()] + sys.meta_path +sys.meta_path = [TornadoImporter(), SixRedirectImporter()] + sys.meta_path # All salt related deprecation warnings should be shown once each! From 5dba916b4af1f850fb4178bc210c266671576130 Mon Sep 17 00:00:00 2001 From: Richard Phillis Date: Tue, 23 Nov 2021 09:05:28 +1000 Subject: [PATCH 692/769] Prevent get_tops from performing a Set operation on a List --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index b1afb20..bf8bc17 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -692,7 +692,7 @@ def get_tops(self): else: saltenvs.add(self.opts["pillarenv"]) else: - saltenvs = self._get_envs() + saltenvs = {*self._get_envs()} if self.opts.get("pillar_source_merging_strategy", None) == "none": saltenvs &= {self.saltenv or "base"} From 9ff151a060f11ac70acb9f26f4e9aac356d8f4fa Mon Sep 17 00:00:00 2001 From: Richard Phillis Date: Wed, 15 Dec 2021 12:56:19 +1000 Subject: [PATCH 693/769] Mutate set rather than constructing a new one As favoured by @waynew --- src/saltext/consul/pillar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index bf8bc17..40645a3 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -692,7 +692,7 @@ def get_tops(self): else: saltenvs.add(self.opts["pillarenv"]) else: - saltenvs = {*self._get_envs()} + saltenvs.update(self._get_envs()) if self.opts.get("pillar_source_merging_strategy", None) == "none": saltenvs &= {self.saltenv or "base"} From c392bfe406f2188283baab3f74c0b60305db1b3b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 23 Sep 2021 20:28:39 -0700 Subject: [PATCH 694/769] Clean up transports --- src/saltext/consul/pillar/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 40645a3..486b1d9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -10,11 +10,11 @@ import sys import traceback +import salt.channel.client import salt.ext.tornado.gen import salt.fileclient import salt.loader import salt.minion -import salt.transport.client import salt.utils.args import salt.utils.cache import salt.utils.crypt @@ -218,7 +218,7 @@ def __init__( self.ext = ext self.grains = grains self.minion_id = minion_id - self.channel = salt.transport.client.AsyncReqChannel.factory(opts) + self.channel = salt.channel.client.AsyncReqChannel.factory(opts) if pillarenv is not None: self.opts["pillarenv"] = pillarenv self.pillar_override = pillar_override or {} @@ -311,7 +311,7 @@ def __init__( self.ext = ext self.grains = grains self.minion_id = minion_id - self.channel = salt.transport.client.ReqChannel.factory(opts) + self.channel = salt.channel.client.ReqChannel.factory(opts) if pillarenv is not None: self.opts["pillarenv"] = pillarenv self.pillar_override = pillar_override or {} From 22f33f1716773a363edd4193972738f4cf5b84b8 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 20 Jan 2022 13:19:58 -0800 Subject: [PATCH 695/769] [merge jam] Master port 49261 - consul modules (#58101) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add consul states and acl function present/absent * add consul to states doc index * refact/fix consul states * fix doc, fix states * fix name parameter for acl_changes * fixing pylint errors * small changes after review by @rallytime * fix header count * Update consul.py * fix acl_exists description, fix when both id and name are missing * Adding some tests for consul module and consul state module. Some additional fixes in the consul module. * Fixing tests. * Fixing failing tests on Windows. * Adding changelog. * Adding some tests for consul module and consul state module. Some additional fixes in the consul module. * moving tests to pytest. * manual black changes. * One more manual black change. * fixing formatting. Adding versionadded for state module. Co-authored-by: Rémi Jouannet Co-authored-by: Mike Place Co-authored-by: Daniel Wozniak Co-authored-by: Wayne Werner --- docs/ref/states/all/salt.states.consul.rst | 6 + src/saltext/consul/modules/consul.py | 44 ++-- src/saltext/consul/states/consul.py | 203 ++++++++++++++++++ tests/pytests/unit/modules/test_consul.py | 235 +++++++++++++++++++++ tests/pytests/unit/states/test_consul.py | 160 ++++++++++++++ 5 files changed, 631 insertions(+), 17 deletions(-) create mode 100644 docs/ref/states/all/salt.states.consul.rst create mode 100644 src/saltext/consul/states/consul.py create mode 100644 tests/pytests/unit/modules/test_consul.py create mode 100644 tests/pytests/unit/states/test_consul.py diff --git a/docs/ref/states/all/salt.states.consul.rst b/docs/ref/states/all/salt.states.consul.rst new file mode 100644 index 0000000..b0934a9 --- /dev/null +++ b/docs/ref/states/all/salt.states.consul.rst @@ -0,0 +1,6 @@ +================== +salt.states.consul +================== + +.. automodule:: salt.states.consul + :members: \ No newline at end of file diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 814bc47..43d7b6a 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -5,7 +5,6 @@ """ - import base64 import http.client import logging @@ -94,10 +93,15 @@ def _query( ret["data"] = result.get("dict", result) ret["res"] = True elif result.get("status", None) == http.client.NO_CONTENT: + ret["data"] = "No content available." ret["res"] = False elif result.get("status", None) == http.client.NOT_FOUND: ret["data"] = "Key not found." ret["res"] = False + elif result.get("error", None): + ret["data"] = "An error occurred." + ret["error"] = result["error"] + ret["res"] = False else: if result: ret["data"] = result @@ -271,7 +275,7 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): query_params = {} available_sessions = session_list(consul_url=consul_url, return_list=True) - _current = get(consul_url=consul_url, key=key) + _current = get(consul_url=consul_url, token=token, key=key) if "flags" in kwargs: if kwargs["flags"] >= 0 and kwargs["flags"] <= 2 ** 64: @@ -323,7 +327,7 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): data = value function = "kv/{}".format(key) method = "PUT" - ret = _query( + res = _query( consul_url=consul_url, token=token, function=function, @@ -332,12 +336,14 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): query_params=query_params, ) - if ret["res"]: + if res["res"]: ret["res"] = True ret["data"] = "Added key {} with value {}.".format(key, value) else: ret["res"] = False ret["data"] = "Unable to add key {} with value {}.".format(key, value) + if "error" in res: + ret["error"] = res["error"] return ret @@ -389,7 +395,7 @@ def delete(consul_url=None, token=None, key=None, **kwargs): return ret function = "kv/{}".format(key) - ret = _query( + res = _query( consul_url=consul_url, token=token, function=function, @@ -397,12 +403,14 @@ def delete(consul_url=None, token=None, key=None, **kwargs): query_params=query_params, ) - if ret["res"]: + if res["res"]: ret["res"] = True ret["message"] = "Deleted key {}.".format(key) else: ret["res"] = False ret["message"] = "Unable to delete key {}.".format(key) + if "error" in res: + ret["error"] = res["error"] return ret @@ -1343,14 +1351,18 @@ def session_destroy(consul_url=None, token=None, session=None, **kwargs): function = "session/destroy/{}".format(session) res = _query( - consul_url=consul_url, function=function, token=token, query_params=query_params + consul_url=consul_url, + function=function, + token=token, + method="PUT", + query_params=query_params, ) if res["res"]: ret["res"] = True - ret["message"] = "Created Service {}.".format(kwargs["name"]) + ret["message"] = "Destroyed Session {}.".format(session) else: ret["res"] = False - ret["message"] = "Unable to create service {}.".format(kwargs["name"]) + ret["message"] = "Unable to destroy session {}.".format(session) return ret @@ -2071,6 +2083,9 @@ def acl_create(consul_url=None, token=None, **kwargs): ret["res"] = False return ret + if "id" in kwargs: + data["id"] = kwargs["id"] + if "name" in kwargs: data["Name"] = kwargs["name"] else: @@ -2155,7 +2170,7 @@ def acl_update(consul_url=None, token=None, **kwargs): ret["message"] = "ACL {} created.".format(kwargs["name"]) else: ret["res"] = False - ret["message"] = "Adding ACL {} failed.".format(kwargs["name"]) + ret["message"] = "Updating ACL {} failed.".format(kwargs["name"]) return ret @@ -2190,7 +2205,7 @@ def acl_delete(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - function = "acl/delete/{}".format(kwargs["id"]) + function = "acl/destroy/{}".format(kwargs["id"]) res = _query( consul_url=consul_url, token=token, data=data, method="PUT", function=function ) @@ -2309,14 +2324,9 @@ def acl_list(consul_url=None, token=None, **kwargs): ret["res"] = False return ret - if "id" not in kwargs: - ret["message"] = 'Required parameter "id" is missing.' - ret["res"] = False - return ret - function = "acl/list" ret = _query( - consul_url=consul_url, token=token, data=data, method="PUT", function=function + consul_url=consul_url, token=token, data=data, method="GET", function=function ) return ret diff --git a/src/saltext/consul/states/consul.py b/src/saltext/consul/states/consul.py new file mode 100644 index 0000000..b35b726 --- /dev/null +++ b/src/saltext/consul/states/consul.py @@ -0,0 +1,203 @@ +""" +Consul Management +================= + +.. versionadded:: 3005 + +The consul module is used to create and manage Consul ACLs + +.. code-block:: yaml + + acl_present: + consul.acl_present: + - id: 38AC8470-4A83-4140-8DFD-F924CD32917F + - name: acl_name + - rules: node "" {policy = "write"} service "" {policy = "read"} key "_rexec" {policy = "write"} + - type: client + - consul_url: http://localhost:8500 + + acl_delete: + consul.acl_absent: + - id: 38AC8470-4A83-4140-8DFD-F924CD32917F +""" +import logging + +log = logging.getLogger(__name__) + + +def _acl_changes(name, id=None, type=None, rules=None, consul_url=None, token=None): + """ + return True if the acl need to be update, False if it doesn't need to be update + """ + info = __salt__["consul.acl_info"](id=id, token=token, consul_url=consul_url) + + if info["res"] and info["data"][0]["Name"] != name: + return True + elif info["res"] and info["data"][0]["Rules"] != rules: + return True + elif info["res"] and info["data"][0]["Type"] != type: + return True + else: + return False + + +def _acl_exists(name=None, id=None, token=None, consul_url=None): + """ + Check the acl exists by using the name or the ID, + name is ignored if ID is specified, + if only Name is used the ID associated with it is returned + """ + + ret = {"result": False, "id": None} + + if id: + info = __salt__["consul.acl_info"](id=id, token=token, consul_url=consul_url) + elif name: + info = __salt__["consul.acl_list"](token=token, consul_url=consul_url) + else: + return ret + + if info.get("data"): + for acl in info["data"]: + if id and acl["ID"] == id: + ret["result"] = True + ret["id"] = id + elif name and acl["Name"] == name: + ret["result"] = True + ret["id"] = acl["ID"] + + return ret + + +def acl_present( + name, + id=None, + token=None, + type="client", + rules="", + consul_url="http://localhost:8500", +): + """ + Ensure the ACL is present + + name + Specifies a human-friendly name for the ACL token. + + id + Specifies the ID of the ACL. + + type: client + Specifies the type of ACL token. Valid values are: client and management. + + rules + Specifies rules for this ACL token. + + consul_url : http://locahost:8500 + consul URL to query + + .. note:: + For more information https://www.consul.io/api/acl.html#create-acl-token, https://www.consul.io/api/acl.html#update-acl-token + """ + + ret = { + "name": name, + "changes": {}, + "result": True, + "comment": 'ACL "{}" exists and is up to date'.format(name), + } + + exists = _acl_exists(name, id, token, consul_url) + + if not exists["result"]: + if __opts__["test"]: + ret["result"] = None + ret["comment"] = "The acl doesn't exist, it will be created" + return ret + + create = __salt__["consul.acl_create"]( + name=name, id=id, token=token, type=type, rules=rules, consul_url=consul_url + ) + if create["res"]: + ret["result"] = True + ret["comment"] = "The acl has been created" + elif not create["res"]: + ret["result"] = False + ret["comment"] = "Failed to create the acl" + elif exists["result"]: + changes = _acl_changes( + name=name, + id=exists["id"], + token=token, + type=type, + rules=rules, + consul_url=consul_url, + ) + if changes: + if __opts__["test"]: + ret["result"] = None + ret["comment"] = "The acl exists and will be updated" + return ret + + update = __salt__["consul.acl_update"]( + name=name, + id=exists["id"], + token=token, + type=type, + rules=rules, + consul_url=consul_url, + ) + if update["res"]: + ret["result"] = True + ret["comment"] = "The acl has been updated" + elif not update["res"]: + ret["result"] = False + ret["comment"] = "Failed to update the acl" + + return ret + + +def acl_absent(name, id=None, token=None, consul_url="http://localhost:8500"): + """ + Ensure the ACL is absent + + name + Specifies a human-friendly name for the ACL token. + + id + Specifies the ID of the ACL. + + token + token to authenticate you Consul query + + consul_url : http://locahost:8500 + consul URL to query + + .. note:: + For more information https://www.consul.io/api/acl.html#delete-acl-token + + """ + ret = { + "name": id, + "changes": {}, + "result": True, + "comment": 'ACL "{}" does not exist'.format(id), + } + + exists = _acl_exists(name, id, token, consul_url) + if exists["result"]: + if __opts__["test"]: + ret["result"] = None + ret["comment"] = "The acl exists, it will be deleted" + return ret + + delete = __salt__["consul.acl_delete"]( + id=exists["id"], token=token, consul_url=consul_url + ) + if delete["res"]: + ret["result"] = True + ret["comment"] = "The acl has been deleted" + elif not delete["res"]: + ret["result"] = False + ret["comment"] = "Failed to delete the acl" + + return ret diff --git a/tests/pytests/unit/modules/test_consul.py b/tests/pytests/unit/modules/test_consul.py new file mode 100644 index 0000000..81b8938 --- /dev/null +++ b/tests/pytests/unit/modules/test_consul.py @@ -0,0 +1,235 @@ +""" +Test case for the consul execution module +""" + + +import logging + +import pytest +import salt.modules.consul as consul +import salt.utils.platform +from salt.exceptions import SaltInvocationError +from tests.support.mock import MagicMock, patch + +log = logging.getLogger(__name__) + + +@pytest.fixture +def configure_loader_modules(): + return { + consul: { + "__opts__": {"consul": {"url": "http://127.0.0.1", "token": "test_token"}}, + "__grains__": {"id": "test-minion"}, + } + } + + +def test_list(): + """ + Test salt.modules.consul.list function + """ + mock_query = MagicMock(return_value={"data": ["foo"], "res": True}) + with patch.object(consul, "_query", mock_query): + consul_return = consul.list_(consul_url="http://127.0.0.1", token="test_token") + + assert consul_return == {"data": ["foo"], "res": True} + + +def test_get(): + """ + Test salt.modules.consul.get function + """ + # + # No key argument results in SaltInvocationError, exception + # + with pytest.raises(SaltInvocationError): + consul.put(consul_url="http://127.0.0.1", token="test_token") + + mock_query = MagicMock( + return_value={ + "data": [ + { + "LockIndex": 0, + "Key": "foo", + "Flags": 0, + "Value": "YmFy", + "CreateIndex": 128, + "ModifyIndex": 128, + }, + ], + "res": True, + } + ) + with patch.object(consul, "_query", mock_query): + consul_return = consul.get( + consul_url="http://127.0.0.1", key="foo", token="test_token" + ) + _expected = { + "data": [ + { + "CreateIndex": 128, + "Flags": 0, + "Key": "foo", + "LockIndex": 0, + "ModifyIndex": 128, + "Value": "YmFy", + } + ], + "res": True, + } + + assert consul_return == _expected + + mock_query = MagicMock( + return_value={ + "data": [ + { + "LockIndex": 0, + "Key": "foo", + "Flags": 0, + "Value": "b'bar'", + "CreateIndex": 128, + "ModifyIndex": 128, + }, + ], + "res": True, + } + ) + with patch.object(consul, "_query", mock_query): + consul_return = consul.get( + consul_url="http://127.0.0.1", key="foo", token="test_token" + ) + _expected = { + "data": [ + { + "CreateIndex": 128, + "Flags": 0, + "Key": "foo", + "LockIndex": 0, + "ModifyIndex": 128, + "Value": "b'bar'", + } + ], + "res": True, + } + + assert consul_return == _expected + + +def test_put(): + """ + Test salt.modules.consul.put function + """ + # + # No key argument results in SaltInvocationError, exception + # + with pytest.raises(SaltInvocationError): + consul.put(consul_url="http://127.0.0.1", token="test_token") + + # + # Test when we're unable to connect to Consul + # + mock_consul_get = { + "data": [ + { + "LockIndex": 0, + "Key": "web/key1", + "Flags": 0, + "Value": "ImhlbGxvIHRoZXJlIg==", + "CreateIndex": 299, + "ModifyIndex": 299, + } + ], + "res": True, + } + with patch.object(consul, "session_list", MagicMock(return_value=[])): + with patch.object(consul, "get", MagicMock(return_value=mock_consul_get)): + ret = consul.put( + consul_url="http://127.0.0.1:8501", + token="test_token", + key="web/key1", + value="Hello world", + ) + expected_res = (False,) + expected_data = "Unable to add key web/key1 with value Hello world." + if salt.utils.platform.is_windows(): + expected_error = "Unknown error" + else: + expected_error = "Connection refused" + assert not ret["res"] + assert expected_data == ret["data"] + assert expected_error in ret["error"] + + # + # Working as expected + # + mock_query = MagicMock( + return_value={ + "data": [ + { + "LockIndex": 0, + "Key": "foo", + "Flags": 0, + "Value": "YmFy", + "CreateIndex": 128, + "ModifyIndex": 128, + }, + ], + "res": True, + } + ) + with patch.object(consul, "session_list", MagicMock(return_value=[])): + with patch.object(consul, "get", MagicMock(return_value=mock_consul_get)): + with patch.object(consul, "_query", mock_query): + ret = consul.put( + consul_url="http://127.0.0.1:8500", + token="test_token", + key="web/key1", + value="Hello world", + ) + _expected = {"res": True, "data": "Added key web/key1 with value Hello world."} + assert ret == _expected + + +def test_delete(): + """ + Test salt.modules.consul.delete function + """ + # + # No key argument results in SaltInvocationError, exception + # + with pytest.raises(SaltInvocationError): + consul.put(consul_url="http://127.0.0.1", token="test_token") + + # + # Test when we're unable to connect to Consul + # + ret = consul.delete( + consul_url="http://127.0.0.1:8501", + token="test_token", + key="web/key1", + value="Hello world", + ) + expected_res = (False,) + expected_data = "Unable to delete key web/key1." + if salt.utils.platform.is_windows(): + expected_error = "Unknown error" + else: + expected_error = "Connection refused" + assert not ret["res"] + assert expected_data == ret["message"] + assert expected_error in ret["error"] + + # + # Working as expected + # + mock_query = MagicMock(return_value={"data": True, "res": True}) + with patch.object(consul, "_query", mock_query): + ret = consul.delete( + consul_url="http://127.0.0.1:8500", + token="test_token", + key="web/key1", + value="Hello world", + ) + _expected = {"res": True, "message": "Deleted key web/key1."} + assert ret == _expected diff --git a/tests/pytests/unit/states/test_consul.py b/tests/pytests/unit/states/test_consul.py new file mode 100644 index 0000000..69e8cb6 --- /dev/null +++ b/tests/pytests/unit/states/test_consul.py @@ -0,0 +1,160 @@ +""" +Test case for the consul state module +""" + + +import logging + +import pytest +import salt.states.consul as consul +from tests.support.mock import MagicMock, patch + +log = logging.getLogger(__name__) + + +@pytest.fixture +def configure_loader_modules(): + return { + consul: { + "__opts__": { + "consul": {"url": "http://127.0.0.1", "token": "test_token"}, + "test": False, + }, + "__grains__": {"id": "test-minion"}, + } + } + + +def test_acl_present(): + """ + Test salt.states.consul.acl_present function + """ + acl_info = { + "data": [ + { + "ID": "1d53dcd6-8f4f-431e-818a-9987996701a1", + "Name": "89530557-8f18-4e29-a2d6-5b2fc8bca713", + "Type": "client", + "Rules": "", + "CreateIndex": 419, + "ModifyIndex": 420, + } + ], + "res": True, + } + + acl_info_mock = MagicMock(return_value=acl_info) + with patch.dict(consul.__salt__, {"consul.acl_info": acl_info_mock}): + with patch.object(consul, "_acl_changes", MagicMock(return_value=False)): + consul_return = consul.acl_present( + "my_acl", + id="1d53dcd6-8f4f-431e-818a-9987996701a1", + token="89530557-8f18-4e29-a2d6-5b2fc8bca713", + type="client", + consul_url="http://localhost:8500", + ) + + _expected = { + "changes": {}, + "comment": 'ACL "my_acl" exists and is up to date', + "name": "my_acl", + "result": True, + } + assert consul_return == _expected + + acl_update_mock = MagicMock( + return_value={ + "data": {"ID": "1D53DCD6-8F4F-431E-818A-9987996701A1"}, + "res": True, + } + ) + with patch.object(consul, "_acl_changes", MagicMock(return_value=True)): + with patch.dict(consul.__salt__, {"consul.acl_update": acl_update_mock}): + consul_return = consul.acl_present( + "my_acl", + id="1d53dcd6-8f4f-431e-818a-9987996701a1", + token="89530557-8f18-4e29-a2d6-5b2fc8bca713", + type="client", + consul_url="http://localhost:8500", + ) + + _expected = { + "changes": {}, + "comment": "The acl has been updated", + "name": "my_acl", + "result": True, + } + assert consul_return == _expected + + +def test_acl_absent(): + """ + Test salt.states.consul.acl_absent function + """ + # + # Test when the ACL does exist + # + acl_info = { + "data": [ + { + "ID": "1d53dcd6-8f4f-431e-818a-9987996701a1", + "Name": "89530557-8f18-4e29-a2d6-5b2fc8bca713", + "Type": "client", + "Rules": "", + "CreateIndex": 419, + "ModifyIndex": 420, + } + ], + "res": True, + } + acl_info_mock = MagicMock(return_value=acl_info) + acl_delete_mock = MagicMock( + return_value={ + "res": True, + "message": "ACL 38AC8470-4A83-4140-8DFD-F924CD32917F deleted.", + } + ) + with patch.dict(consul.__salt__, {"consul.acl_info": acl_info_mock}): + with patch.dict(consul.__salt__, {"consul.acl_delete": acl_delete_mock}): + consul_return = consul.acl_absent( + "my_acl", + id="1d53dcd6-8f4f-431e-818a-9987996701a1", + token="89530557-8f18-4e29-a2d6-5b2fc8bca713", + consul_url="http://localhost:8500", + ) + + _expected = { + "changes": {}, + "comment": "The acl has been deleted", + "name": "1d53dcd6-8f4f-431e-818a-9987996701a1", + "result": True, + } + assert consul_return == _expected + + # + # Test when the ACL does not exist + # + acl_info = {"data": [], "res": True} + acl_info_mock = MagicMock(return_value=acl_info) + acl_delete_mock = MagicMock( + return_value={ + "res": True, + "message": "ACL 38AC8470-4A83-4140-8DFD-F924CD32917F deleted.", + } + ) + with patch.dict(consul.__salt__, {"consul.acl_info": acl_info_mock}): + with patch.dict(consul.__salt__, {"consul.acl_delete": acl_delete_mock}): + consul_return = consul.acl_absent( + "my_acl", + id="1d53dcd6-8f4f-431e-818a-9987996701a1", + token="89530557-8f18-4e29-a2d6-5b2fc8bca713", + consul_url="http://localhost:8500", + ) + + _expected = { + "changes": {}, + "comment": 'ACL "1d53dcd6-8f4f-431e-818a-9987996701a1" does not exist', + "name": "1d53dcd6-8f4f-431e-818a-9987996701a1", + "result": True, + } + assert consul_return == _expected From 75e8ef640a32a9e656a3733efc51c0f13b02dec2 Mon Sep 17 00:00:00 2001 From: Cesar Augusto Sanchez Date: Sat, 29 Aug 2020 12:19:56 -0400 Subject: [PATCH 696/769] fix(consul): serialize to JSON only non string objects. Fixes 35215 --- src/saltext/consul/modules/consul.py | 12 +- tests/pytests/unit/modules/test_consul.py | 1651 +++++++++++++++++++++ 2 files changed, 1658 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 43d7b6a..8f70c86 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -74,9 +74,11 @@ def _query( if method == "GET": data = None else: - if data is None: - data = {} - data = salt.utils.json.dumps(data) + if data is not None: + if type(data) != str: + data = salt.utils.json.dumps(data) + else: + data = salt.utils.json.dumps({}) result = salt.utils.http.query( url, @@ -2293,7 +2295,7 @@ def acl_clone(consul_url=None, token=None, **kwargs): if res["res"]: ret["res"] = True ret["message"] = "ACL {} cloned.".format(kwargs["name"]) - ret["ID"] = ret["data"] + ret["ID"] = res["data"] else: ret["res"] = False ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) @@ -2388,7 +2390,7 @@ def event_fire(consul_url=None, token=None, name=None, **kwargs): if res["res"]: ret["res"] = True ret["message"] = "Event {} fired.".format(name) - ret["data"] = ret["data"] + ret["data"] = res["data"] else: ret["res"] = False ret["message"] = "Cloning ACL item {} failed.".format(kwargs["name"]) diff --git a/tests/pytests/unit/modules/test_consul.py b/tests/pytests/unit/modules/test_consul.py index 81b8938..700997a 100644 --- a/tests/pytests/unit/modules/test_consul.py +++ b/tests/pytests/unit/modules/test_consul.py @@ -7,6 +7,8 @@ import pytest import salt.modules.consul as consul +import salt.utils.http +import salt.utils.json import salt.utils.platform from salt.exceptions import SaltInvocationError from tests.support.mock import MagicMock, patch @@ -233,3 +235,1652 @@ def test_delete(): ) _expected = {"res": True, "message": "Deleted key web/key1."} assert ret == _expected + + +def test_agent_maintenance(): + """ + Test consul agent maintenance + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_maintenance(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required argument + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = 'Required parameter "enable" is missing.' + result = consul.agent_maintenance(consul_url=consul_url) + expected = {"message": msg, "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Agent maintenance mode {}ed." + value = "enabl" + result = consul.agent_maintenance(consul_url=consul_url, enable=value) + expected = {"message": msg.format(value), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to change maintenance mode for agent." + value = "enabl" + result = consul.agent_maintenance(consul_url=consul_url, enable=value) + expected = {"message": msg, "res": True} + assert expected == result + + +def test_agent_join(): + """ + Test consul agent join + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_join(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required argument + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = 'Required parameter "address" is missing.' + pytest.raises( + SaltInvocationError, consul.agent_join, consul_url=consul_url + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Agent joined the cluster" + result = consul.agent_join(consul_url=consul_url, address="test") + expected = {"message": msg, "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to join the cluster." + value = "enabl" + result = consul.agent_join(consul_url=consul_url, address="test") + expected = {"message": msg, "res": False} + assert expected == result + + +def test_agent_leave(): + """ + Test consul agent leave + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_join(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + node = "node1" + + # no required argument + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, consul.agent_leave, consul_url=consul_url + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Node {} put in leave state." + result = consul.agent_leave(consul_url=consul_url, node=node) + expected = {"message": msg.format(node), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to change state for {}." + result = consul.agent_leave(consul_url=consul_url, node=node) + expected = {"message": msg.format(node), "res": False} + assert expected == result + + +def test_agent_check_register(): + """ + Test consul agent check register + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_check_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + name = "name1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_check_register, + consul_url=consul_url, + ) + + # missing script, or http + msg = 'Required parameter "script" or "http" is missing.' + result = consul.agent_check_register(consul_url=consul_url, name=name) + expected = {"message": msg, "res": False} + assert expected == result + + # missing interval + msg = 'Required parameter "interval" is missing.' + result = consul.agent_check_register( + consul_url=consul_url, + name=name, + script="test", + http="test", + ttl="test", + ) + expected = {"message": msg, "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Check {} added to agent." + result = consul.agent_check_register( + consul_url=consul_url, + name=name, + script="test", + http="test", + ttl="test", + interval="test", + ) + expected = {"message": msg.format(name), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to add check to agent." + result = consul.agent_check_register( + consul_url=consul_url, + name=name, + script="test", + http="test", + ttl="test", + interval="test", + ) + expected = {"message": msg.format(name), "res": False} + assert expected == result + + +def test_agent_check_deregister(): + """ + Test consul agent check register + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_check_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + checkid = "id1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_check_deregister, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Check {} removed from agent." + result = consul.agent_check_deregister( + consul_url=consul_url, checkid=checkid + ) + expected = {"message": msg.format(checkid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to remove check from agent." + result = consul.agent_check_deregister( + consul_url=consul_url, checkid=checkid + ) + expected = {"message": msg.format(checkid), "res": False} + assert expected == result + + +def test_agent_check_pass(): + """ + Test consul agent check pass + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_check_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + checkid = "id1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_check_pass, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Check {} marked as passing." + result = consul.agent_check_pass(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to update check {}." + result = consul.agent_check_pass(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": False} + assert expected == result + + +def test_agent_check_warn(): + """ + Test consul agent check warn + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_check_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + checkid = "id1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_check_warn, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Check {} marked as warning." + result = consul.agent_check_warn(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to update check {}." + result = consul.agent_check_warn(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": False} + assert expected == result + + +def test_agent_check_fail(): + """ + Test consul agent check warn + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_check_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + checkid = "id1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_check_fail, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Check {} marked as critical." + result = consul.agent_check_fail(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to update check {}." + result = consul.agent_check_fail(consul_url=consul_url, checkid=checkid) + expected = {"message": msg.format(checkid), "res": False} + assert expected == result + + +def test_agent_service_register(): + """ + Test consul agent service register + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_service_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + name = "name1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_service_register, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Service {} registered on agent." + result = consul.agent_service_register( + consul_url=consul_url, + name=name, + script="test", + http="test", + ttl="test", + interval="test", + ) + expected = {"message": msg.format(name), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to register service {}." + result = consul.agent_service_register( + consul_url=consul_url, + name=name, + script="test", + http="test", + ttl="test", + interval="test", + ) + expected = {"message": msg.format(name), "res": False} + assert expected == result + + +def test_agent_service_deregister(): + """ + Test consul agent service deregister + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_service_deregister(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + serviceid = "sid1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_service_deregister, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Service {} removed from agent." + result = consul.agent_service_deregister( + consul_url=consul_url, serviceid=serviceid + ) + expected = {"message": msg.format(serviceid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to remove service {}." + result = consul.agent_service_deregister( + consul_url=consul_url, serviceid=serviceid + ) + expected = {"message": msg.format(serviceid), "res": False} + assert expected == result + + +def test_agent_service_maintenance(): + """ + Test consul agent service maintenance + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.agent_service_maintenance(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + serviceid = "sid1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.agent_service_maintenance, + consul_url=consul_url, + ) + + # missing enable + msg = 'Required parameter "enable" is missing.' + result = consul.agent_service_maintenance( + consul_url=consul_url, serviceid=serviceid + ) + expected = {"message": msg, "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Service {} set in maintenance mode." + result = consul.agent_service_maintenance( + consul_url=consul_url, serviceid=serviceid, enable=True + ) + expected = {"message": msg.format(serviceid), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to set service {} to maintenance mode." + result = consul.agent_service_maintenance( + consul_url=consul_url, serviceid=serviceid, enable=True + ) + expected = {"message": msg.format(serviceid), "res": False} + assert expected == result + + +def test_session_create(): + """ + Test consul session create + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + result = consul.session_create(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + name = "name1" + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + pytest.raises( + SaltInvocationError, + consul.session_create, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Created session {}." + result = consul.session_create(consul_url=consul_url, name=name) + expected = {"message": msg.format(name), "res": True} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result_false): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + with patch.object( + salt.modules.consul, "session_list", return_value=mock_result + ): + msg = "Unable to create session {}." + result = consul.session_create(consul_url=consul_url, name=name) + expected = {"message": msg.format(name), "res": False} + assert expected == result + + +def test_session_list(): + """ + Test consul session list + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.session_list(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.session_list(consul_url=consul_url) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_session_destroy(): + """ + Test consul session destroy + """ + consul_url = "http://localhost:1313" + key = "cluster/key" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + session = "sid1" + name = "test" + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.session_destroy(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.session_destroy, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + msg = "Destroyed Session {}." + result = consul.session_destroy( + consul_url=consul_url, session=session, name="test" + ) + expected = {"message": msg.format(session), "res": True} + assert expected == result + + +def test_session_info(): + """ + Test consul session info + """ + consul_url = "http://localhost:1313" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + session = "sid1" + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.session_info(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.session_info, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.session_info(consul_url=consul_url, session=session) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_catalog_register(): + """ + Test consul catalog register + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + node = "node1" + address = "addres1" + nodemeta = { + "Cpu": "blah", + "Cpu_num": "8", + "Memory": "1024", + "Os": "rhel8", + "Osarch": "x86_64", + "Kernel": "foo.bar", + "Kernelrelease": "foo.release", + "localhost": "localhost", + "nodename": node, + "os_family": "adams", + "lsb_distrib_description": "distro", + "master": "master", + } + nodemeta_kwargs = { + "cpu": "blah", + "num_cpus": "8", + "mem": "1024", + "oscode": "rhel8", + "osarch": "x86_64", + "kernel": "foo.bar", + "kernelrelease": "foo.release", + "localhost": "localhost", + "nodename": node, + "os_family": "adams", + "lsb_distrib_description": "distro", + "master": "master", + } + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_register(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_register(consul_url=consul_url, token=token) + expected = { + "message": "Required argument node argument is missing.", + "res": False, + } + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_register( + consul_url=consul_url, + token=token, + node=node, + address=address, + **nodemeta_kwargs + ) + expected = { + "data": {"Address": address, "Node": node, "NodeMeta": nodemeta}, + "message": "Catalog registration for {} successful.".format(node), + "res": True, + } + + assert expected == result + + +def test_catalog_deregister(): + """ + Test consul catalog deregister + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + node = "node1" + address = "addres1" + serviceid = "server1" + checkid = "check1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_deregister(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_deregister(consul_url=consul_url, token=token) + expected = {"message": "Node argument required.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_deregister( + consul_url=consul_url, + token=token, + node=node, + serviceid=serviceid, + checkid=checkid, + ) + expected = { + "message": "Catalog item {} removed.".format(node), + "res": True, + } + + assert expected == result + + +def test_catalog_datacenters(): + """ + Test consul catalog datacenters + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_datacenters(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_datacenters(consul_url=consul_url, token=token) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_catalog_nodes(): + """ + Test consul catalog nodes + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_nodes(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_nodes(consul_url=consul_url, token=token) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_catalog_services(): + """ + Test consul catalog services + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_services(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_services(consul_url=consul_url, token=token) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_catalog_service(): + """ + Test consul catalog service + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + service = "service" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_service(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.catalog_service, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_service( + consul_url=consul_url, token=token, service=service + ) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_catalog_node(): + """ + Test consul catalog node + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + node = "node" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.catalog_node(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.catalog_node, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.catalog_node(consul_url=consul_url, token=token, node=node) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_health_node(): + """ + Test consul health node + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + node = "node" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.health_node(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.health_node, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.health_node(consul_url=consul_url, token=token, node=node) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_health_checks(): + """ + Test consul health checks + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + service = "service" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.health_checks(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.health_checks, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.health_checks( + consul_url=consul_url, token=token, service=service + ) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_health_service(): + """ + Test consul health service + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + service = "service" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.health_service(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.health_service, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.health_service( + consul_url=consul_url, token=token, service=service + ) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_health_state(): + """ + Test consul health state + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + state = "state" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.health_state(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.health_state, + consul_url=consul_url, + ) + + # state not in allowed states + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.health_state( + consul_url=consul_url, token=token, state=state + ) + expected = { + "message": "State must be any, unknown, passing, warning, or critical.", + "res": False, + } + assert expected == result + + state = "warning" + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.health_state( + consul_url=consul_url, token=token, state=state + ) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_status_leader(): + """ + Test consul status leader + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.status_leader(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.status_leader(consul_url=consul_url, token=token) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_status_peers(): + """ + Test consul status peers + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.status_peers(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.status_peers(consul_url=consul_url, token=token) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_acl_create(): + """ + Test consul acl create + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_create(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.acl_create, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_create(consul_url=consul_url, token=token, name=name) + expected = {"message": "ACL {} created.".format(name), "res": True} + assert expected == result + + +def test_acl_update(): + """ + Test consul acl update + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + aclid = "id1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_update(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_update(consul_url=consul_url) + expected = { + "message": 'Required parameter "id" is missing.', + "res": False, + } + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.acl_update, + consul_url=consul_url, + id=aclid, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_update( + consul_url=consul_url, token=token, name=name, id=aclid + ) + expected = {"message": "ACL {} created.".format(name), "res": True} + assert expected == result + + +def test_acl_delete(): + """ + Test consul acl delete + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + aclid = "id1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_delete(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_delete(consul_url=consul_url) + expected = { + "message": 'Required parameter "id" is missing.', + "res": False, + } + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_delete( + consul_url=consul_url, token=token, name=name, id=aclid + ) + expected = {"message": "ACL {} deleted.".format(aclid), "res": True} + assert expected == result + + +def test_acl_info(): + """ + Test consul acl info + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + aclid = "id1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_info(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_info(consul_url=consul_url) + expected = { + "message": 'Required parameter "id" is missing.', + "res": False, + } + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_info( + consul_url=consul_url, token=token, name=name, id=aclid + ) + expected = {"data": "test", "res": True} + assert expected == result + + +def test_acl_clone(): + """ + Test consul acl clone + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + aclid = "id1" + + mock_result = aclid + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_clone(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_clone(consul_url=consul_url) + expected = { + "message": 'Required parameter "id" is missing.', + "res": False, + } + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_clone( + consul_url=consul_url, token=token, name=name, id=aclid + ) + expected = { + "ID": aclid, + "message": "ACL {} cloned.".format(name), + "res": True, + } + assert expected == result + + +def test_acl_list(): + """ + Test consul acl list + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + aclid = "id1" + + mock_result = aclid + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.acl_list(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.acl_list( + consul_url=consul_url, token=token, name=name, id=aclid + ) + expected = {"data": "id1", "res": True} + assert expected == result + + +def test_event_fire(): + """ + Test consul event fire + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.event_fire(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.event_fire, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.event_fire(consul_url=consul_url, token=token, name=name) + expected = { + "data": "test", + "message": "Event {} fired.".format(name), + "res": True, + } + assert expected == result + + +def test_event_list(): + """ + Test consul event list + """ + consul_url = "http://localhost:1313" + token = "randomtoken" + name = "name1" + + mock_result = "test" + mock_http_result = {"status": 200, "dict": mock_result} + mock_http_result_false = {"status": 204, "dict": mock_result} + mock_url = MagicMock(return_value=consul_url) + mock_nourl = MagicMock(return_value=None) + + # no consul url error + with patch.dict(consul.__salt__, {"config.get": mock_nourl}): + result = consul.event_list(consul_url="") + expected = {"message": "No Consul URL found.", "res": False} + assert expected == result + + # no required arguments + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + pytest.raises( + SaltInvocationError, + consul.event_list, + consul_url=consul_url, + ) + + with patch.object(salt.utils.http, "query", return_value=mock_http_result): + with patch.dict(consul.__salt__, {"config.get": mock_url}): + result = consul.event_list(consul_url=consul_url, token=token, name=name) + expected = {"data": "test", "res": True} + assert expected == result From 332dd29ad6771ad378d34367a7d8338e74f2b33e Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Thu, 27 Jan 2022 08:05:13 -0500 Subject: [PATCH 697/769] add __env__ substitution inside file and pillar root paths --- src/saltext/consul/pillar/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 486b1d9..25b2135 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -657,6 +657,11 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): env, ) opts["pillar_roots"].pop("__env__") + for env in opts["pillar_roots"]: + for idx, root in enumerate(opts["pillar_roots"][env]): + opts["pillar_roots"][env][idx] = opts["pillar_roots"][env][idx].replace( + "__env__", env + ) return opts def _get_envs(self): From 9a3a49c4049bafb6b3fd1a6fa135e2d316599857 Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Fri, 28 Jan 2022 22:50:37 -0500 Subject: [PATCH 698/769] refactor to only substitute __env__ environment --- src/saltext/consul/pillar/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 25b2135..4ae6bb9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -651,17 +651,16 @@ def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None): env, ) opts["pillar_roots"][env] = opts["pillar_roots"].pop("__env__") + for idx, root in enumerate(opts["pillar_roots"][env]): + opts["pillar_roots"][env][idx] = opts["pillar_roots"][env][ + idx + ].replace("__env__", env) else: log.debug( "pillar_roots __env__ ignored (environment '%s' found in pillar_roots)", env, ) opts["pillar_roots"].pop("__env__") - for env in opts["pillar_roots"]: - for idx, root in enumerate(opts["pillar_roots"][env]): - opts["pillar_roots"][env][idx] = opts["pillar_roots"][env][idx].replace( - "__env__", env - ) return opts def _get_envs(self): From a6a1babafc0f25687a6364471f855030bd419d44 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 24 Feb 2022 16:04:58 +0000 Subject: [PATCH 699/769] Delete the `cachedir` on each test Signed-off-by: Pedro Algarvio --- tests/pytests/functional/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 1e81032..d1f985b 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,4 +1,5 @@ import logging +import pathlib import shutil import pytest @@ -79,6 +80,11 @@ def loaders(minion_opts): @pytest.fixture(autouse=True) def reset_loaders_state(loaders): + # Delete the files cache after each test + cachedir = pathlib.Path(loaders.opts["cachedir"]) + shutil.rmtree(str(cachedir), ignore_errors=True) + cachedir.mkdir(parents=True, exist_ok=True) + # The above can be deleted after pytest-salt-factories>=1.0.0rc7 has been merged try: # Run the tests yield From a19615dd62174341c0265acbc77f2a8c8bdb0504 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 25 Feb 2022 07:47:48 +0000 Subject: [PATCH 700/769] Reduce proxy minions fixture scope. * Make sure to also delete the dummy delta proxy minions keys * Use ``salt_master.{pillar,state}_tree`` instead of direct use of the ``temp_file`` helper. Signed-off-by: Pedro Algarvio --- tests/pytests/integration/conftest.py | 74 --------------------------- 1 file changed, 74 deletions(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 09485bb..21c3cc8 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -49,80 +49,6 @@ def salt_sub_minion(salt_master, salt_sub_minion_factory): yield salt_sub_minion_factory -@pytest.fixture(scope="package") -def salt_proxy(salt_master, salt_proxy_factory): - """ - A running salt-proxy fixture - """ - assert salt_master.is_running() - with salt_proxy_factory.started(): - yield salt_proxy_factory - - -@pytest.fixture(scope="package") -def deltaproxy_pillar_tree(base_env_pillar_tree_root_dir, salt_delta_proxy_factory): - """ - Create the pillar files for controlproxy and two dummy proxy minions - """ - top_file = """ - base: - '{}': - - controlproxy - dummy_proxy_one: - - dummy_proxy_one - dummy_proxy_two: - - dummy_proxy_two - """.format( - salt_delta_proxy_factory.id - ) - controlproxy_pillar_file = """ - proxy: - proxytype: deltaproxy - ids: - - dummy_proxy_one - - dummy_proxy_two - """ - - dummy_proxy_one_pillar_file = """ - proxy: - proxytype: dummy - """ - - dummy_proxy_two_pillar_file = """ - proxy: - proxytype: dummy - """ - - top_tempfile = pytest.helpers.temp_file( - "top.sls", top_file, base_env_pillar_tree_root_dir - ) - controlproxy_tempfile = pytest.helpers.temp_file( - "controlproxy.sls", controlproxy_pillar_file, base_env_pillar_tree_root_dir - ) - dummy_proxy_one_tempfile = pytest.helpers.temp_file( - "dummy_proxy_one.sls", - dummy_proxy_one_pillar_file, - base_env_pillar_tree_root_dir, - ) - dummy_proxy_two_tempfile = pytest.helpers.temp_file( - "dummy_proxy_two.sls", - dummy_proxy_two_pillar_file, - base_env_pillar_tree_root_dir, - ) - with top_tempfile, controlproxy_tempfile, dummy_proxy_one_tempfile, dummy_proxy_two_tempfile: - yield - - -@pytest.fixture(scope="package") -def salt_delta_proxy(salt_master, salt_delta_proxy_factory, deltaproxy_pillar_tree): - """ - A running salt-proxy fixture - """ - assert salt_master.is_running() - with salt_delta_proxy_factory.started(): - yield salt_delta_proxy_factory - - @pytest.fixture(scope="package") def salt_cli(salt_master): """ From f093bcac36a6faf2882e98a73b61521ca5cfdbb8 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Thu, 24 Feb 2022 11:23:31 -0700 Subject: [PATCH 701/769] Remove enable_slsvars_fixes feature flag and enable behavior by default --- tests/pytests/functional/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index d1f985b..cdcb4af 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -62,7 +62,6 @@ def minion_opts( { "file_client": "local", "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, - "features": {"enable_slsvars_fixes": True}, } ) factory = salt_factories.salt_minion_daemon( From a7946a4a9a04cd80b2b59bc50652456e03f38adf Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 28 Mar 2022 18:51:18 +0100 Subject: [PATCH 702/769] Clean `LD_LIBRARY_PATH` when shelling out under a tiamat package Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0a18a9b..2de0c58 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -134,3 +134,7 @@ def __define_global_system_encoding_variable__(): # Import Salt's logging machinery import salt._logging.impl # isort:skip pylint: disable=unused-import + +# Do any nessessary patching when salt is running from a tiamat package +# This code is temporary and will exist until we can handle this on salt-pkg +import salt.utils.tiamatpkg # isort:skip pylint: disable=unused-import From a7ff9fb86a86f99104a7571fbf61098837bfe85d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 29 Mar 2022 09:57:53 +0100 Subject: [PATCH 703/769] Switch to proving the proper hooks to PyInstaller Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 2de0c58..0a18a9b 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -134,7 +134,3 @@ def __define_global_system_encoding_variable__(): # Import Salt's logging machinery import salt._logging.impl # isort:skip pylint: disable=unused-import - -# Do any nessessary patching when salt is running from a tiamat package -# This code is temporary and will exist until we can handle this on salt-pkg -import salt.utils.tiamatpkg # isort:skip pylint: disable=unused-import From 457d3072b8fe8b44f1f547b18415af3a9c3ee8ad Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Fri, 8 Apr 2022 08:18:38 -0500 Subject: [PATCH 704/769] Cache API tests & unifying behavior (#61229) * Add tests for cache backends * Tests run against everything! * Get consul passing cache tests * now mysql tests also pass * comments * add mysql cache tests * Ensure consul cache is flushing timestamp keys * Ensure etcd cache is flushing timestamp keys * Update redis cache api to conform to localfs There was a bug in some of the lookups where it was looking at sub banks instead of the banks containing the leaf/key nodes. That was fixed. Also restored the original behaviors, only adding the updated functionality, namely adding timestamp key/values to redis. * Fix etcd path inconsistency Previously on X-Files. Wait nope. Previously on etcd cache files... :joy: if a path were 1/2/3 then etcd would break with IndexError, but when it was 1/2/3/4/5 then instead of returning the "file" (key?) 5 it would return `4/5` as the name. Which is probably not correct, and definitely inconsistent with localfs (and every other cache module). Now it will always only return the final segment of the cache (e.g. 7 for 1/2/3/4/5/6/7 and 5 for 1/2/5) * Remove comments and cleanup test setup Comments were outdated, now they're gone or fixed. Also shifted around some of the test setup to make it a bit cleaner. * Add localfs backed memcache to the tests Fixed an inconsistency when flushing banks with None key. Also added to the test run. We may want to add other backing caches via alternate fixtures? * Bad serial * No this is wrong * Probably the right lazy loader fix * Restore tested mysql cache functionality Added a force_reconnect function because I couldn't figure out how to remove the client from the context any other way. There may be a better way to do that, but there will probably be some loader changes anyway. * remove silly files Saltfile never should have been added in the first place and the integration test wasn't really necessary, functional was better/lighter. * Fix redis_cache cluster mode import Fixes #60272 - the import statement was totally incorrect. Now it will actually attempt to connect to a redis cluster. * payload should string-ify all the data * Fix unit tests for parametrization From what I can tell, previously the query was being built in Python instead of using mysql to add the parameters, and this probably horked up the handling of binary strings. Unless some other version of the DB will be problematic... * Fix missing docstring complaints * incorporate PR feedback * Skip docker if not found and add the rest of mysql I really really hate the way this is configured but I can't make pytest use fixtures as parameters in fixtures. I may have to struggle to come up with another way. * Tests all passing, woo! * run black * Disable pylint here * Skip when mysql fails during CI This has been tested locally and it runs all of them. I'm sure with the CI environment it's possible that it fails for no particularly good reason. And it's definitely not worth blocking a PR for (though arguably it could be if *all* of the images fail, but...) --- src/saltext/consul/cache/__init__.py | 9 +- src/saltext/consul/cache/consul.py | 60 ++++++++--- tests/pytests/functional/cache/__init__.py | 0 tests/pytests/functional/conftest.py | 110 +++++++++++++++++++++ 4 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 tests/pytests/functional/cache/__init__.py diff --git a/src/saltext/consul/cache/__init__.py b/src/saltext/consul/cache/__init__.py index 464493e..2b13a36 100644 --- a/src/saltext/consul/cache/__init__.py +++ b/src/saltext/consul/cache/__init__.py @@ -39,7 +39,7 @@ class Cache: The name of the cache driver to use. This is the name of the python module of the `salt.cache` package. Default is `localfs`. - Terminology. + Terminology: Salt cache subsystem is organized as a tree with nodes and leafs like a filesystem. Cache consists of banks. Each bank can contain a number of @@ -345,5 +345,10 @@ def store(self, bank, key, data): self.storage[(bank, key)] = [time.time(), data] def flush(self, bank, key=None): - self.storage.pop((bank, key), None) + if key is None: + for bank_, key_ in tuple(self.storage): + if bank == bank_: + self.storage.pop((bank_, key_)) + else: + self.storage.pop((bank, key), None) super().flush(bank, key) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index fe5a266..14c08cc 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -3,6 +3,10 @@ .. versionadded:: 2016.11.2 +.. versionchanged:: 3005.0 + + Timestamp/cache updated support added. + :depends: python-consul >= 0.2.0 It is up to the system administrator to set up and configure the Consul @@ -30,6 +34,12 @@ consul.consistency: default consul.dc: dc1 consul.verify: True + consul.timestamp_suffix: .tstamp # Added in 3005.0 + +In order to bring the cache APIs into conformity, in 3005.0 timestamp +information gets stored as a separate ``{key}.tstamp`` key/value. If your +existing functionality depends on being able to store normal keys with the +``.tstamp`` suffix, override the ``consul.timestamp_suffix`` default config. Related docs could be found in the `python-consul documentation`_. @@ -47,6 +57,7 @@ """ import logging +import time import salt.payload from salt.exceptions import SaltCacheError @@ -61,6 +72,7 @@ log = logging.getLogger(__name__) api = None +_tstamp_suffix = ".tstamp" # Define the module's virtual name @@ -90,7 +102,8 @@ def __virtual__(): } try: - global api + global api, _tstamp_suffix + _tstamp_suffix = __opts__.get("consul.timestamp_suffix", _tstamp_suffix) api = consul.Consul(**consul_kwargs) except AttributeError: return ( @@ -107,9 +120,12 @@ def store(bank, key, data): Store a key value. """ c_key = "{}/{}".format(bank, key) + tstamp_key = "{}/{}{}".format(bank, key, _tstamp_suffix) + try: c_data = salt.payload.dumps(data) api.kv.put(c_key, c_data) + api.kv.put(tstamp_key, salt.payload.dumps(int(time.time()))) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( "There was an error writing the key, {}: {}".format(c_key, exc) @@ -138,9 +154,13 @@ def flush(bank, key=None): """ if key is None: c_key = bank + tstamp_key = None else: c_key = "{}/{}".format(bank, key) + tstamp_key = "{}/{}{}".format(bank, key, _tstamp_suffix) try: + if tstamp_key: + api.kv.delete(tstamp_key) return api.kv.delete(c_key, recurse=key is None) except Exception as exc: # pylint: disable=broad-except raise SaltCacheError( @@ -166,7 +186,7 @@ def list_(bank): out = set() for key in keys: out.add(key[len(bank) + 1 :].rstrip("/")) - keys = list(out) + keys = [o for o in out if not o.endswith(_tstamp_suffix)] return keys @@ -174,14 +194,28 @@ def contains(bank, key): """ Checks if the specified bank contains the specified key. """ - if key is None: - return True # any key could be a branch and a leaf at the same time in Consul - else: - try: - c_key = "{}/{}".format(bank, key) - _, value = api.kv.get(c_key) - except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error getting the key, {}: {}".format(c_key, exc) - ) - return value is not None + try: + c_key = "{}/{}".format(bank, key or "") + _, value = api.kv.get(c_key, keys=True) + except Exception as exc: # pylint: disable=broad-except + raise SaltCacheError( + "There was an error getting the key, {}: {}".format(c_key, exc) + ) + return value is not None + + +def updated(bank, key): + """ + Return the Unix Epoch timestamp of when the key was last updated. Return + None if key is not found. + """ + c_key = "{}/{}{}".format(bank, key, _tstamp_suffix) + try: + _, value = api.kv.get(c_key) + if value is None: + return None + return salt.payload.loads(value["Value"]) + except Exception as exc: # pylint: disable=broad-except + raise SaltCacheError( + "There was an error reading the key, {}: {}".format(c_key, exc) + ) diff --git a/tests/pytests/functional/cache/__init__.py b/tests/pytests/functional/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index cdcb4af..6e840d1 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -3,8 +3,17 @@ import shutil import pytest +from saltfactories.daemons.container import Container from saltfactories.utils.functional import Loaders +try: + import docker +except ImportError: + # Test suites depending on docker should be using + # docker = pytest.importorskip("docker") + # so any fixtures using docker shouldn't ever be called or used. + pass + log = logging.getLogger(__name__) @@ -90,3 +99,104 @@ def reset_loaders_state(loaders): finally: # Reset the loaders state loaders.reset_state() + + +@pytest.fixture(scope="module") +def docker_client(): + try: + client = docker.from_env() + except docker.errors.DockerException: + pytest.skip("Failed to get a connection to docker running on the system") + connectable = Container.client_connectable(client) + if connectable is not True: # pragma: nocover + pytest.skip(connectable) + return client + + +def pull_or_skip(image_name, docker_client): + try: + docker_client.images.pull(image_name) + except docker.errors.APIError as exc: + pytest.skip("Failed to pull docker image {!r}: {}".format(image_name, exc)) + except ImportError: + pytest.skip("docker module was not installed") + return image_name + + +@pytest.fixture(scope="module") +def docker_redis_image(docker_client): + return pull_or_skip("redis:alpine", docker_client) + + +@pytest.fixture(scope="module") +def docker_consul_image(docker_client): + return pull_or_skip("consul:latest", docker_client) + + +# Pytest does not have the ability to parametrize fixtures with parametriezed +# fixtures, which is super annoying. In other words, in order to have a `cache` +# test fixture that takes different versions of the cache that depend on +# different docker images, I've gotta make up fixtures for each +# image+container. When https://github.com/pytest-dev/pytest/issues/349 is +# actually fixed then we can go ahead and refactor all of these mysql images +# (and container fixtures depending on it) into a single fixture. + + +@pytest.fixture(scope="module") +def docker_mysql_5_6_image(docker_client): + return pull_or_skip("mysql/mysql-server:5.6", docker_client) + + +@pytest.fixture(scope="module") +def docker_mysql_5_7_image(docker_client): + return pull_or_skip("mysql/mysql-server:5.7", docker_client) + + +@pytest.fixture(scope="module") +def docker_mysql_8_0_image(docker_client): + return pull_or_skip("mysql/mysql-server:8.0", docker_client) + + +@pytest.fixture(scope="module") +def docker_mariadb_10_1_image(docker_client): + return pull_or_skip("mariadb:10.1", docker_client) + + +@pytest.fixture(scope="module") +def docker_mariadb_10_2_image(docker_client): + return pull_or_skip("mariadb:10.2", docker_client) + + +@pytest.fixture(scope="module") +def docker_mariadb_10_3_image(docker_client): + return pull_or_skip("mariadb:10.3", docker_client) + + +@pytest.fixture(scope="module") +def docker_mariadb_10_4_image(docker_client): + return pull_or_skip("mariadb:10.4", docker_client) + + +@pytest.fixture(scope="module") +def docker_mariadb_10_5_image(docker_client): + return pull_or_skip("mariadb:10.5", docker_client) + + +@pytest.fixture(scope="module") +def docker_percona_5_5_image(docker_client): + return pull_or_skip("percona:5.5", docker_client) + + +@pytest.fixture(scope="module") +def docker_percona_5_6_image(docker_client): + return pull_or_skip("percona:5.6", docker_client) + + +@pytest.fixture(scope="module") +def docker_percona_5_7_image(docker_client): + return pull_or_skip("percona:5.7", docker_client) + + +@pytest.fixture(scope="module") +def docker_percona_8_0_image(docker_client): + return pull_or_skip("percona:8.0", docker_client) From 05ede4c1521c15f56f8ebdd1a80aebba381c2887 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 28 Jun 2021 12:07:24 +0100 Subject: [PATCH 705/769] Add the new refactored salt logging. It also handles multiprocessing logging. --- src/saltext/consul/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 0a18a9b..ebc54ac 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -132,5 +132,9 @@ def __define_global_system_encoding_variable__(): # This is now garbage collectable del __define_global_system_encoding_variable__ -# Import Salt's logging machinery -import salt._logging.impl # isort:skip pylint: disable=unused-import +# Make sure Salt's logging tweaks are always present +# DO NOT MOVE THIS IMPORT +# pylint: disable=unused-import +import salt._logging # isort:skip + +# pylint: enable=unused-import From 9e7d72521cf8200cfb417baf5857fdb33e1206ca Mon Sep 17 00:00:00 2001 From: Daniel Wozniak Date: Mon, 18 Apr 2022 04:14:51 -0700 Subject: [PATCH 706/769] Merge forward from 3004.1 (#61888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Redirect imports of ``salt.ext.six`` to ``six`` Fixes #60966 * Latest changelog update for 3004 * Handle signals and properly exit, instead of raising exceptions. This was introduced in 26fcda50740c92fe3e4a5ca50a0d83227009942a Fixes #60391 Fixes #60963 * Add test for #61003 * Fix #61003 Restored the previously shifted check for version_to_remove in old[target]. This had been extracted along with the correctly extracted double pkg_params[target] lookup, but that lost the `target in old` guard. Putting the check back here prevents KeyError when looking for a non-existent target in `old`. * Handle various architecture formats in aptpkg module * Write file even if does not exist * only run test on debian based platforms * remove extra space for arch * convert pathlib to string for pkgrepo test * Use temporary files first then copy to sources files * fixes saltstack/salt#59182 fix handling of duplicate keys in rest_cherrypy data * added changelog * remove log messages to prevent leaks of sensitive info * Reverting changes in PR #60150. Updating installed and removed functions to return changes when test=True. * Adding changelog. * Add a test and fix for extra-filerefs * Do not break master_tops for minion with version lower to 3003 * Add changelog file * Add extra comment to clarify discussion * Update changelog file * Add deprecated changelog * Assert that the command didn't finish Refs https://github.com/saltstack/salt/pull/60972 * Always restore signals, even when exceptions occur * Reset signal handlers before starting the process * Make sure that the `ProcessManager` doesn't always ignore signals * Provide valid default value for bootstrap_delay * Update changelog for 3004 * Update changelog and release notes for 3004 * Add PR 61020 to changelog * Change MD5 to SHA256 fingerprint for new github.com fingerprint * Check only ssh-rsa encyption for set_known_host * Use main branch for kitchen-docker project * Add tests for validate_tgt This function evolved over the years, but never had any tests. We're adding tests now to cover the various cases: - there are no valid minions (currently fails, should return False) - there are target minions that aren't in valid minions (correctly fails) - target minions are a subset of valid minions (i.e. all of the target minions are found in the valid minions -- there are no extras) (correctly passes) * Refactor minions should be a subset of v_minions - the extra code was just getting in the way. Also, this function evolved over time but the docstring never kept up. Updated the docstring to more accurately describe the function's behavior. * Fix #60413 When using a syndic and user auth, it was possible for v_minions and minions to be two empty sets, which returned True. This allowed the user to still publish the function. The Syndic would get the published event and apply it, even though it should have been rejected. However, if there are no valid minions, then it doesn't matter what the targets are -- there are not valid targets, so there's no reason to do any further checks. * Rename changelog to security * add cve# to changelog * Sign pillar data * Add regression tests for CVE-2022-22934 * Add changelog for cve-2022-22934 * Provide users with a nice warning when something goes wrong * Rename changelog file * Fix wart in tests * Return bool when using m2crypo * Limit the amount of empty space while searching ifconfig output * Update changelog/cve-2020-22937.security Co-authored-by: Megan Wilhite * Prevent auth replays and sign replies * Add tests for cve-2022-22935 * Add changelog for cve-2020-22935 * Fix typo * Prevent replays of file server requests * Add regresion tests for fileserver nonce * Add changelog for cve-2022-22936 * Job replay mitigation * Fix merge warts * more test fixes * Fix auth tests on windows * Remove unwanted requirements change * Clean up cruft * update docs for 3004.1 release * Fix warts in new minion auth * Test fix * Update release notes * Remove cve from non cve worty issue * Add serial to payload in publisher process * Fix channel tests Fix broken channel tests by populating an AES key and serial. * Windows test fix * windows tests plz work Co-authored-by: Pedro Algarvio Co-authored-by: ScriptAutomate Co-authored-by: Wayne Werner Co-authored-by: Megan Wilhite Co-authored-by: nicholasmhughes Co-authored-by: Gareth J. Greenaway Co-authored-by: Pablo Suárez Hernández Co-authored-by: Alyssa Rock Co-authored-by: krionbsd Co-authored-by: Megan Wilhite Co-authored-by: Frode Gundersen Co-authored-by: MKLeb --- src/saltext/consul/pillar/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 4ae6bb9..7be9635 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -9,6 +9,7 @@ import os import sys import traceback +import uuid import salt.channel.client import salt.ext.tornado.gen @@ -262,6 +263,9 @@ def compile_pillar(self): load, dictkey="pillar", ) + except salt.crypt.AuthenticationError as exc: + log.error(exc.message) + raise SaltClientError("Exception getting pillar.") except Exception: # pylint: disable=broad-except log.exception("Exception getting pillar:") raise SaltClientError("Exception getting pillar.") From 11dd21b1cb7d2607f68af9c77ac401d58db8760f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 31 May 2022 17:35:38 +0100 Subject: [PATCH 707/769] Updates required for ``pytest-salt-factories>=1.0.0rc16`` Signed-off-by: Pedro Algarvio --- tests/pytests/functional/conftest.py | 6 ------ tests/pytests/integration/conftest.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 6e840d1..30edd9a 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,5 +1,4 @@ import logging -import pathlib import shutil import pytest @@ -88,11 +87,6 @@ def loaders(minion_opts): @pytest.fixture(autouse=True) def reset_loaders_state(loaders): - # Delete the files cache after each test - cachedir = pathlib.Path(loaders.opts["cachedir"]) - shutil.rmtree(str(cachedir), ignore_errors=True) - cachedir.mkdir(parents=True, exist_ok=True) - # The above can be deleted after pytest-salt-factories>=1.0.0rc7 has been merged try: # Run the tests yield diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 21c3cc8..0a2fd09 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -31,7 +31,7 @@ def salt_minion(salt_master, salt_minion_factory): # Sync All salt_call_cli = salt_minion_factory.salt_call_cli() ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.exitcode == 0, ret + assert ret.returncode == 0, ret yield salt_minion_factory @@ -45,7 +45,7 @@ def salt_sub_minion(salt_master, salt_sub_minion_factory): # Sync All salt_call_cli = salt_sub_minion_factory.salt_call_cli() ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.exitcode == 0, ret + assert ret.returncode == 0, ret yield salt_sub_minion_factory From 45e69c990d2c2f5da096eab5a59b0b316acb094c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 30 May 2022 03:08:28 +0100 Subject: [PATCH 708/769] Switch cache functional tests to the new container support code in salt-factories Additionally, split the tests into their own modules and clean up unused, or no longer necessary code. Signed-off-by: Pedro Algarvio --- tests/pytests/functional/cache/test_consul.py | 70 +++++++++++ tests/pytests/functional/conftest.py | 110 ------------------ 2 files changed, 70 insertions(+), 110 deletions(-) create mode 100644 tests/pytests/functional/cache/test_consul.py diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py new file mode 100644 index 0000000..78d3aab --- /dev/null +++ b/tests/pytests/functional/cache/test_consul.py @@ -0,0 +1,70 @@ +import logging +import socket +import time + +import pytest +import salt.cache +import salt.loader +from saltfactories.utils import random_string +from tests.pytests.functional.cache.helpers import run_common_cache_tests + +docker = pytest.importorskip("docker") + +log = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("dockerd"), +] + + +def confirm_consul_is_ready(timeout_at, container): + sleeptime = 0.1 + consul_port = container.get_host_port_binding(8500, protocol="tcp") + while time.time() <= timeout_at: + time.sleep(sleeptime) + try: + with socket.create_connection(("localhost", consul_port), timeout=1) as cli: + cli.send(b"GET /v1/kv/fnord HTTP/1.1\n\n") + cli.recv(2048) + break + except ConnectionResetError as e: + if e.errno == 104: + sleeptime += sleeptime + else: + return False + return True + + +@pytest.fixture(scope="module") +def consul_container(salt_factories): + + container = salt_factories.get_container( + random_string("consul-server-"), + image_name="consul:latest", + container_run_kwargs={"ports": {"8500/tcp": None}}, + pull_before_start=True, + skip_on_pull_failure=True, + skip_if_docker_client_not_connectable=True, + ) + container.container_start_check(confirm_consul_is_ready, container) + + with container.started() as factory: + yield factory + + +@pytest.fixture +def cache(minion_opts, consul_container): + opts = minion_opts.copy() + opts["cache"] = "consul" + opts["consul.host"] = "127.0.0.1" + opts["consul.port"] = consul_container.get_host_port_binding(8500, protocol="tcp") + # NOTE: If you would like to ensure that alternate suffixes are properly + # tested, simply change this value and re-run the tests. + opts["consul.timestamp_suffix"] = ".frobnosticate" + cache = salt.cache.factory(opts) + return cache + + +def test_caching(subtests, cache): + run_common_cache_tests(subtests, cache) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 30edd9a..7acd8a4 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -2,17 +2,8 @@ import shutil import pytest -from saltfactories.daemons.container import Container from saltfactories.utils.functional import Loaders -try: - import docker -except ImportError: - # Test suites depending on docker should be using - # docker = pytest.importorskip("docker") - # so any fixtures using docker shouldn't ever be called or used. - pass - log = logging.getLogger(__name__) @@ -93,104 +84,3 @@ def reset_loaders_state(loaders): finally: # Reset the loaders state loaders.reset_state() - - -@pytest.fixture(scope="module") -def docker_client(): - try: - client = docker.from_env() - except docker.errors.DockerException: - pytest.skip("Failed to get a connection to docker running on the system") - connectable = Container.client_connectable(client) - if connectable is not True: # pragma: nocover - pytest.skip(connectable) - return client - - -def pull_or_skip(image_name, docker_client): - try: - docker_client.images.pull(image_name) - except docker.errors.APIError as exc: - pytest.skip("Failed to pull docker image {!r}: {}".format(image_name, exc)) - except ImportError: - pytest.skip("docker module was not installed") - return image_name - - -@pytest.fixture(scope="module") -def docker_redis_image(docker_client): - return pull_or_skip("redis:alpine", docker_client) - - -@pytest.fixture(scope="module") -def docker_consul_image(docker_client): - return pull_or_skip("consul:latest", docker_client) - - -# Pytest does not have the ability to parametrize fixtures with parametriezed -# fixtures, which is super annoying. In other words, in order to have a `cache` -# test fixture that takes different versions of the cache that depend on -# different docker images, I've gotta make up fixtures for each -# image+container. When https://github.com/pytest-dev/pytest/issues/349 is -# actually fixed then we can go ahead and refactor all of these mysql images -# (and container fixtures depending on it) into a single fixture. - - -@pytest.fixture(scope="module") -def docker_mysql_5_6_image(docker_client): - return pull_or_skip("mysql/mysql-server:5.6", docker_client) - - -@pytest.fixture(scope="module") -def docker_mysql_5_7_image(docker_client): - return pull_or_skip("mysql/mysql-server:5.7", docker_client) - - -@pytest.fixture(scope="module") -def docker_mysql_8_0_image(docker_client): - return pull_or_skip("mysql/mysql-server:8.0", docker_client) - - -@pytest.fixture(scope="module") -def docker_mariadb_10_1_image(docker_client): - return pull_or_skip("mariadb:10.1", docker_client) - - -@pytest.fixture(scope="module") -def docker_mariadb_10_2_image(docker_client): - return pull_or_skip("mariadb:10.2", docker_client) - - -@pytest.fixture(scope="module") -def docker_mariadb_10_3_image(docker_client): - return pull_or_skip("mariadb:10.3", docker_client) - - -@pytest.fixture(scope="module") -def docker_mariadb_10_4_image(docker_client): - return pull_or_skip("mariadb:10.4", docker_client) - - -@pytest.fixture(scope="module") -def docker_mariadb_10_5_image(docker_client): - return pull_or_skip("mariadb:10.5", docker_client) - - -@pytest.fixture(scope="module") -def docker_percona_5_5_image(docker_client): - return pull_or_skip("percona:5.5", docker_client) - - -@pytest.fixture(scope="module") -def docker_percona_5_6_image(docker_client): - return pull_or_skip("percona:5.6", docker_client) - - -@pytest.fixture(scope="module") -def docker_percona_5_7_image(docker_client): - return pull_or_skip("percona:5.7", docker_client) - - -@pytest.fixture(scope="module") -def docker_percona_8_0_image(docker_client): - return pull_or_skip("percona:8.0", docker_client) From 9c9b45fb6b238664ba335b1ac8c7e59ce8c03f63 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 17 Jun 2022 22:28:00 +0100 Subject: [PATCH 709/769] Namespace the functional tests loaders Signed-off-by: Pedro Algarvio --- tests/pytests/functional/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 7acd8a4..9bf6daf 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -73,7 +73,7 @@ def minion_opts( @pytest.fixture(scope="module") def loaders(minion_opts): - return Loaders(minion_opts) + return Loaders(minion_opts, loaded_base_name="{}.loaded".format(__name__)) @pytest.fixture(autouse=True) From 6a21386e6f70f1bf493b31cc05e5191895950901 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 25 Jul 2022 17:27:02 +0100 Subject: [PATCH 710/769] Switch to the github repository container and our own container mirrors Signed-off-by: Pedro Algarvio --- tests/pytests/functional/cache/test_consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py index 78d3aab..6680a75 100644 --- a/tests/pytests/functional/cache/test_consul.py +++ b/tests/pytests/functional/cache/test_consul.py @@ -41,7 +41,7 @@ def consul_container(salt_factories): container = salt_factories.get_container( random_string("consul-server-"), - image_name="consul:latest", + image_name="ghcr.io/saltstack/salt-ci-containers/consul:latest", container_run_kwargs={"ports": {"8500/tcp": None}}, pull_before_start=True, skip_on_pull_failure=True, From 988eaf594fc4c16ecff887463c8b2136f8c07a4e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Jul 2022 10:42:30 +0100 Subject: [PATCH 711/769] Update to isort 5.10.1 Signed-off-by: Pedro Algarvio --- src/saltext/consul/pillar/__init__.py | 2 +- tests/pytests/functional/cache/test_consul.py | 3 ++- tests/pytests/unit/modules/test_consul.py | 1 + tests/pytests/unit/states/test_consul.py | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7be9635..470b4f9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -1140,8 +1140,8 @@ def ext_pillar(self, pillar, errors=None): # the git ext_pillar() func is run, but only for masterless. if self.ext and "git" in self.ext and self.opts.get("__role") != "minion": # Avoid circular import - import salt.utils.gitfs import salt.pillar.git_pillar + import salt.utils.gitfs git_pillar = salt.utils.gitfs.GitPillar( self.opts, diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py index 6680a75..1156b7a 100644 --- a/tests/pytests/functional/cache/test_consul.py +++ b/tests/pytests/functional/cache/test_consul.py @@ -3,9 +3,10 @@ import time import pytest +from saltfactories.utils import random_string + import salt.cache import salt.loader -from saltfactories.utils import random_string from tests.pytests.functional.cache.helpers import run_common_cache_tests docker = pytest.importorskip("docker") diff --git a/tests/pytests/unit/modules/test_consul.py b/tests/pytests/unit/modules/test_consul.py index 700997a..52f1c8e 100644 --- a/tests/pytests/unit/modules/test_consul.py +++ b/tests/pytests/unit/modules/test_consul.py @@ -6,6 +6,7 @@ import logging import pytest + import salt.modules.consul as consul import salt.utils.http import salt.utils.json diff --git a/tests/pytests/unit/states/test_consul.py b/tests/pytests/unit/states/test_consul.py index 69e8cb6..0236745 100644 --- a/tests/pytests/unit/states/test_consul.py +++ b/tests/pytests/unit/states/test_consul.py @@ -6,6 +6,7 @@ import logging import pytest + import salt.states.consul as consul from tests.support.mock import MagicMock, patch From bea77ce1be2df2fe0f37ff4b6e2b2e3ba397109c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 20 Jul 2022 13:50:28 +0100 Subject: [PATCH 712/769] Format source code using latest black Signed-off-by: Pedro Algarvio --- src/saltext/consul/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/modules/consul.py b/src/saltext/consul/modules/consul.py index 8f70c86..205cfb2 100644 --- a/src/saltext/consul/modules/consul.py +++ b/src/saltext/consul/modules/consul.py @@ -280,7 +280,7 @@ def put(consul_url=None, token=None, key=None, value=None, **kwargs): _current = get(consul_url=consul_url, token=token, key=key) if "flags" in kwargs: - if kwargs["flags"] >= 0 and kwargs["flags"] <= 2 ** 64: + if kwargs["flags"] >= 0 and kwargs["flags"] <= 2**64: query_params["flags"] = kwargs["flags"] if "cas" in kwargs: From 94d9cd09dda798915f0882b73d6da8f006f92d70 Mon Sep 17 00:00:00 2001 From: Caleb Beard <53276404+MKLeb@users.noreply.github.com> Date: Sun, 25 Sep 2022 20:28:42 -0400 Subject: [PATCH 713/769] migrate pillar unit tests to pytest (#62745) --- .../pytests/unit/pillar/test_consul_pillar.py | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 tests/pytests/unit/pillar/test_consul_pillar.py diff --git a/tests/pytests/unit/pillar/test_consul_pillar.py b/tests/pytests/unit/pillar/test_consul_pillar.py new file mode 100644 index 0000000..430f7aa --- /dev/null +++ b/tests/pytests/unit/pillar/test_consul_pillar.py @@ -0,0 +1,164 @@ +import pytest + +import salt.pillar.consul_pillar as consul_pillar +from tests.support.mock import MagicMock, patch + +pytestmark = [ + pytest.mark.skipif( + not consul_pillar.consul, reason="python-consul module not installed" + ) +] + + +@pytest.fixture +def base_pillar_data(): + return [ + { + "Value": "/path/to/certs/testsite1.crt", + "Key": "test-shared/sites/testsite1/ssl/certs/SSLCertificateFile", + }, + { + "Value": "/path/to/certs/testsite1.key", + "Key": "test-shared/sites/testsite1/ssl/certs/SSLCertificateKeyFile", + }, + {"Value": None, "Key": "test-shared/sites/testsite1/ssl/certs/"}, + {"Value": "True", "Key": "test-shared/sites/testsite1/ssl/force"}, + {"Value": None, "Key": "test-shared/sites/testsite1/ssl/"}, + { + "Value": "salt://sites/testsite1.tmpl", + "Key": "test-shared/sites/testsite1/template", + }, + {"Value": "test.example.com", "Key": "test-shared/sites/testsite1/uri"}, + {"Value": None, "Key": "test-shared/sites/testsite1/"}, + {"Value": None, "Key": "test-shared/sites/"}, + {"Value": "Test User", "Key": "test-shared/user/full_name"}, + {"Value": "adm\nwww-data\nmlocate", "Key": "test-shared/user/groups"}, + {"Value": '"adm\nwww-data\nmlocate"', "Key": "test-shared/user/dontsplit"}, + {"Value": "yaml:\n key: value\n", "Key": "test-shared/user/dontexpand"}, + {"Value": None, "Key": "test-shared/user/blankvalue"}, + {"Value": "test", "Key": "test-shared/user/login"}, + {"Value": None, "Key": "test-shared/user/"}, + ] + + +@pytest.fixture +def configure_loader_modules(): + return { + consul_pillar: { + "__opts__": { + "consul_config": {"consul.port": 8500, "consul.host": "172.17.0.15"} + }, + "get_conn": MagicMock(return_value="consul_connection"), + } + } + + +def test_connection(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + consul_pillar.ext_pillar( + "testminion", {}, "consul_config root=test-shared/" + ) + consul_pillar.get_conn.assert_called_once_with( + consul_pillar.__opts__, "consul_config" + ) + + +def test_pillar_data(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + pillar_data = consul_pillar.ext_pillar( + "testminion", {}, "consul_config root=test-shared/" + ) + consul_pillar.consul_fetch.assert_called_once_with( + "consul_connection", "test-shared" + ) + assert sorted(pillar_data) == ["sites", "user"] + assert "blankvalue" not in pillar_data["user"] + + +def test_blank_root(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + pillar_data = consul_pillar.ext_pillar("testminion", {}, "consul_config") + consul_pillar.consul_fetch.assert_called_once_with("consul_connection", "") + assert sorted(pillar_data) == ["test-shared"] + + +def test_pillar_nest(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + pillar_data = consul_pillar.ext_pillar( + "testminion", + {}, + "consul_config pillar_root=nested-key/ root=test-shared/ ", + ) + assert sorted(pillar_data["nested-key"]) == ["sites", "user"] + assert "blankvalue" not in pillar_data["nested-key"]["user"] + + +def test_value_parsing(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + pillar_data = consul_pillar.ext_pillar( + "testminion", {}, "consul_config root=test-shared/" + ) + assert isinstance(pillar_data["user"]["dontsplit"], str) + + +def test_non_expansion(base_pillar_data): + with patch.dict( + consul_pillar.__salt__, {"grains.get": MagicMock(return_value=({}))} + ): + with patch.object( + consul_pillar, + "consul_fetch", + MagicMock(return_value=("2232", base_pillar_data)), + ): + pillar_data = consul_pillar.ext_pillar( + "testminion", + {}, + "consul_config root=test-shared/ expand_keys=false", + ) + assert isinstance(pillar_data["user"]["dontexpand"], str) + + +def test_dict_merge(): + test_dict = {} + simple_dict = {"key1": {"key2": "val1"}} + with patch.dict(test_dict, simple_dict): + assert consul_pillar.dict_merge(test_dict, simple_dict) == simple_dict + with patch.dict(test_dict, {"key1": {"key3": {"key4": "value"}}}): + assert consul_pillar.dict_merge(test_dict, simple_dict) == { + "key1": {"key2": "val1", "key3": {"key4": "value"}} + } From 3ff068dacd64f0c55e83d19b4879d738d8aebb99 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 11 Oct 2022 21:19:21 +0100 Subject: [PATCH 714/769] Disable the setuptools user warning "Setuptools is replacing distutils." Once we stop relying on distutils, this filter can be removed. Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ebc54ac..ec551ab 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -69,6 +69,14 @@ def load_module(self, name): append=True, ) +# Filter the setuptools UserWarning until we stop relying on distutils +warnings.filterwarnings( + "ignore", + message="Setuptools is replacing distutils.", + category=UserWarning, + module="_distutils_hack", +) + def __define_global_system_encoding_variable__(): import sys From 5762d1907870f3cbce9026f01cfe4f2e82537c6a Mon Sep 17 00:00:00 2001 From: Victor Zhestkov Date: Mon, 3 Oct 2022 14:47:25 +0300 Subject: [PATCH 715/769] Pass __context__ to ext pillar --- src/saltext/consul/pillar/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 470b4f9..7626689 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -46,6 +46,7 @@ def get_pillar( pillarenv=None, extra_minion_data=None, clean_cache=False, + context=None, ): """ Return the correct pillar driver based on the file_client option @@ -81,6 +82,7 @@ def get_pillar( pillar_override=pillar_override, pillarenv=pillarenv, clean_cache=clean_cache, + context=context, ) return ptype( opts, @@ -92,6 +94,7 @@ def get_pillar( pillar_override=pillar_override, pillarenv=pillarenv, extra_minion_data=extra_minion_data, + context=context, ) @@ -309,6 +312,7 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, + context=None, ): self.opts = opts self.opts["saltenv"] = saltenv @@ -333,6 +337,7 @@ def __init__( merge_lists=True, ) self._closing = False + self.context = context def compile_pillar(self): """ @@ -406,6 +411,7 @@ def __init__( pillarenv=None, extra_minion_data=None, clean_cache=False, + context=None, ): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -432,6 +438,8 @@ def __init__( minion_cache_path=self._minion_cache_path(minion_id), ) + self.context = context + def _minion_cache_path(self, minion_id): """ Return the path to the cache file for the minion. @@ -455,6 +463,7 @@ def fetch_pillar(self): functions=self.functions, pillar_override=self.pillar_override, pillarenv=self.pillarenv, + context=self.context, ) return fresh_pillar.compile_pillar() @@ -530,6 +539,7 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, + context=None, ): self.minion_id = minion_id self.ext = ext @@ -568,7 +578,9 @@ def __init__( if opts.get("pillar_source_merging_strategy"): self.merge_strategy = opts["pillar_source_merging_strategy"] - self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) + self.ext_pillars = salt.loader.pillars( + ext_pillar_opts, self.functions, context=context + ) self.ignored_pillars = {} self.pillar_override = pillar_override or {} if not isinstance(self.pillar_override, dict): From 10950450c65e4917a34dffb11873918744996e65 Mon Sep 17 00:00:00 2001 From: Caleb Beard <53276404+MKLeb@users.noreply.github.com> Date: Wed, 19 Oct 2022 21:43:32 -0400 Subject: [PATCH 716/769] Revert "Pass context to pillar ext" --- src/saltext/consul/pillar/__init__.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 7626689..470b4f9 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -46,7 +46,6 @@ def get_pillar( pillarenv=None, extra_minion_data=None, clean_cache=False, - context=None, ): """ Return the correct pillar driver based on the file_client option @@ -82,7 +81,6 @@ def get_pillar( pillar_override=pillar_override, pillarenv=pillarenv, clean_cache=clean_cache, - context=context, ) return ptype( opts, @@ -94,7 +92,6 @@ def get_pillar( pillar_override=pillar_override, pillarenv=pillarenv, extra_minion_data=extra_minion_data, - context=context, ) @@ -312,7 +309,6 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, - context=None, ): self.opts = opts self.opts["saltenv"] = saltenv @@ -337,7 +333,6 @@ def __init__( merge_lists=True, ) self._closing = False - self.context = context def compile_pillar(self): """ @@ -411,7 +406,6 @@ def __init__( pillarenv=None, extra_minion_data=None, clean_cache=False, - context=None, ): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -438,8 +432,6 @@ def __init__( minion_cache_path=self._minion_cache_path(minion_id), ) - self.context = context - def _minion_cache_path(self, minion_id): """ Return the path to the cache file for the minion. @@ -463,7 +455,6 @@ def fetch_pillar(self): functions=self.functions, pillar_override=self.pillar_override, pillarenv=self.pillarenv, - context=self.context, ) return fresh_pillar.compile_pillar() @@ -539,7 +530,6 @@ def __init__( pillar_override=None, pillarenv=None, extra_minion_data=None, - context=None, ): self.minion_id = minion_id self.ext = ext @@ -578,9 +568,7 @@ def __init__( if opts.get("pillar_source_merging_strategy"): self.merge_strategy = opts["pillar_source_merging_strategy"] - self.ext_pillars = salt.loader.pillars( - ext_pillar_opts, self.functions, context=context - ) + self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} self.pillar_override = pillar_override or {} if not isinstance(self.pillar_override, dict): From a6b664f201f7b79705b4aaf902f126e1cfb5d520 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 16 Sep 2022 16:57:57 +0100 Subject: [PATCH 717/769] `minion_opts` fixture is now available for all tests under `tests/pytests/unit` Signed-off-by: Pedro Algarvio --- tests/pytests/functional/conftest.py | 13 ++++++++++--- tests/pytests/unit/conftest.py | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 tests/pytests/unit/conftest.py diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 9bf6daf..15a880d 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -42,8 +42,8 @@ def minion_config_defaults(): @pytest.fixture(scope="module") def minion_config_overrides(): """ - Functional test modules can provide this fixture to tweak the configuration overrides dictionary - passed to the minion factory + Functional test modules can provide this fixture to tweak the configuration + overrides dictionary passed to the minion factory """ return {} @@ -60,7 +60,14 @@ def minion_opts( minion_config_overrides.update( { "file_client": "local", - "file_roots": {"base": [str(state_tree)], "prod": [str(state_tree_prod)]}, + "file_roots": { + "base": [ + str(state_tree), + ], + "prod": [ + str(state_tree_prod), + ], + }, } ) factory = salt_factories.salt_minion_daemon( diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py new file mode 100644 index 0000000..dda21df --- /dev/null +++ b/tests/pytests/unit/conftest.py @@ -0,0 +1,18 @@ +import pytest + +import salt.config + + +@pytest.fixture +def minion_opts(tmp_path): + """ + Default minion configuration with relative temporary paths to not require root permissions. + """ + opts = salt.config.DEFAULT_MINION_OPTS.copy() + opts["root_dir"] = str(tmp_path) + for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): + dirpath = tmp_path / name + dirpath.mkdir() + opts[name] = str(dirpath) + opts["log_file"] = "logs/minion.log" + return opts From ecb8049197ef710032c0baa88a4ae68438fc16c8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 16 Sep 2022 17:05:09 +0100 Subject: [PATCH 718/769] Add `master_opts` fixture under `tests/pytests/unit` Signed-off-by: Pedro Algarvio --- tests/pytests/unit/conftest.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index dda21df..43deeaa 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -8,11 +8,30 @@ def minion_opts(tmp_path): """ Default minion configuration with relative temporary paths to not require root permissions. """ + root_dir = tmp_path / "minion" opts = salt.config.DEFAULT_MINION_OPTS.copy() - opts["root_dir"] = str(tmp_path) + opts["__role"] = "minion" + opts["root_dir"] = str(root_dir) for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): - dirpath = tmp_path / name - dirpath.mkdir() + dirpath = root_dir / name + dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" return opts + + +@pytest.fixture +def master_opts(tmp_path): + """ + Default master configuration with relative temporary paths to not require root permissions. + """ + root_dir = tmp_path / "master" + opts = salt.config.DEFAULT_MASTER_OPTS.copy() + opts["__role"] = "master" + opts["root_dir"] = str(root_dir) + for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): + dirpath = root_dir / name + dirpath.mkdir(parents=True) + opts[name] = str(dirpath) + opts["log_file"] = "logs/master.log" + return opts From b114d22bfaaaffd6adfa93123c4558031e22c1f6 Mon Sep 17 00:00:00 2001 From: jeanluc Date: Mon, 5 Dec 2022 22:43:13 +0100 Subject: [PATCH 719/769] Make pillar cache pass extra minion data as well --- src/saltext/consul/pillar/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 470b4f9..5a3f538 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -81,6 +81,7 @@ def get_pillar( pillar_override=pillar_override, pillarenv=pillarenv, clean_cache=clean_cache, + extra_minion_data=extra_minion_data, ) return ptype( opts, @@ -419,6 +420,7 @@ def __init__( self.pillar_override = pillar_override self.pillarenv = pillarenv self.clean_cache = clean_cache + self.extra_minion_data = extra_minion_data if saltenv is None: self.saltenv = "base" @@ -455,6 +457,7 @@ def fetch_pillar(self): functions=self.functions, pillar_override=self.pillar_override, pillarenv=self.pillarenv, + extra_minion_data=self.extra_minion_data, ) return fresh_pillar.compile_pillar() From 681aa2baae66c7ad8bd47594f9f91d5c0faa985c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 27 Dec 2022 09:14:00 +0000 Subject: [PATCH 720/769] Implement `exec_module` to remove the deprecation warnings from output `DeprecationWarning: the load_module() method is deprecated and slated for removal in Python 3.12; use exec_module() instead` Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ec551ab..8b72c9e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -35,6 +35,12 @@ def load_module(self, name): sys.modules[name] = mod return mod + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + return None + class SixRedirectImporter: def find_module(self, module_name, package_path=None): @@ -47,6 +53,12 @@ def load_module(self, name): sys.modules[name] = mod return mod + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + return None + # Try our importer first sys.meta_path = [TornadoImporter(), SixRedirectImporter()] + sys.meta_path From 8373120927d9e1ac9e5671fb8eb5fc4c40f71ae4 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 29 Dec 2022 18:06:22 +0000 Subject: [PATCH 721/769] Stop using `salt.config.DEFAULT_{MASTER,MINION}_CONFIG` on unit and functional tests Signed-off-by: Pedro Algarvio --- tests/pytests/functional/conftest.py | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 15a880d..835464e 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -9,7 +9,7 @@ @pytest.fixture(scope="package") def minion_id(): - return "func-tests-minion" + return "func-tests-minion-opts" @pytest.fixture(scope="module") @@ -78,6 +78,53 @@ def minion_opts( return factory.config.copy() +@pytest.fixture(scope="module") +def master_config_defaults(): + """ + Functional test modules can provide this fixture to tweak the default configuration dictionary + passed to the master factory + """ + return {} + + +@pytest.fixture(scope="module") +def master_config_overrides(): + """ + Functional test modules can provide this fixture to tweak the configuration + overrides dictionary passed to the master factory + """ + return {} + + +@pytest.fixture(scope="module") +def master_opts( + salt_factories, + state_tree, + state_tree_prod, + master_config_defaults, + master_config_overrides, +): + master_config_overrides.update( + { + "file_client": "local", + "file_roots": { + "base": [ + str(state_tree), + ], + "prod": [ + str(state_tree_prod), + ], + }, + } + ) + factory = salt_factories.salt_master_daemon( + "func-tests-master-opts", + defaults=master_config_defaults or None, + overrides=master_config_overrides, + ) + return factory.config.copy() + + @pytest.fixture(scope="module") def loaders(minion_opts): return Loaders(minion_opts, loaded_base_name="{}.loaded".format(__name__)) From a5371b5da6a7911e4fbf6e8f7bd4a3eb7381c638 Mon Sep 17 00:00:00 2001 From: Wayne Werner Date: Tue, 13 Dec 2022 13:15:23 -0600 Subject: [PATCH 722/769] Put back deprecation warnings Checked the https://docs.python.org/3/library/warnings.html#warnings.filterwarnings docs but there wasn't any obvious reason why our warnings disappear. This restores the DeprecationWarnings showing up on the command line. --- src/saltext/consul/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8b72c9e..ff7be78 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -70,7 +70,8 @@ def exec_module(self, module): "", # No deprecation message match DeprecationWarning, # This filter is for DeprecationWarnings r"^(salt|salt\.(.*))$", # Match module(s) 'salt' and 'salt.' - append=True, + # Do *NOT* add append=True here - if we do, salt's DeprecationWarnings will + # never show up ) # Filter the backports package UserWarning about being re-imported From 78c9c52f90f69d0ac6e8ee44a2674ac80363fa4b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Dec 2022 15:59:21 +0000 Subject: [PATCH 723/769] Increase timeout to see if the test starts passing Signed-off-by: Pedro Algarvio --- tests/pytests/integration/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytests/integration/conftest.py b/tests/pytests/integration/conftest.py index 0a2fd09..de99d98 100644 --- a/tests/pytests/integration/conftest.py +++ b/tests/pytests/integration/conftest.py @@ -55,7 +55,7 @@ def salt_cli(salt_master): The ``salt`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.salt_cli() + return salt_master.salt_cli(timeout=30) @pytest.fixture(scope="package") @@ -64,7 +64,7 @@ def salt_call_cli(salt_minion): The ``salt-call`` CLI as a fixture against the running minion """ assert salt_minion.is_running() - return salt_minion.salt_call_cli() + return salt_minion.salt_call_cli(timeout=30) @pytest.fixture(scope="package") @@ -73,7 +73,7 @@ def salt_cp_cli(salt_master): The ``salt-cp`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.salt_cp_cli() + return salt_master.salt_cp_cli(timeout=30) @pytest.fixture(scope="package") @@ -82,7 +82,7 @@ def salt_key_cli(salt_master): The ``salt-key`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.salt_key_cli() + return salt_master.salt_key_cli(timeout=30) @pytest.fixture(scope="package") @@ -91,7 +91,7 @@ def salt_run_cli(salt_master): The ``salt-run`` CLI as a fixture against the running master """ assert salt_master.is_running() - return salt_master.salt_run_cli() + return salt_master.salt_run_cli(timeout=30) @pytest.fixture(scope="module") From 86dc1c44294ede80159ff35047e067c0ec222b47 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 6 Feb 2023 19:32:11 +0000 Subject: [PATCH 724/769] Add a few more nice to have hooks Signed-off-by: Pedro Algarvio --- docs/ref/states/all/salt.states.consul.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/states/all/salt.states.consul.rst b/docs/ref/states/all/salt.states.consul.rst index b0934a9..a28a69c 100644 --- a/docs/ref/states/all/salt.states.consul.rst +++ b/docs/ref/states/all/salt.states.consul.rst @@ -3,4 +3,4 @@ salt.states.consul ================== .. automodule:: salt.states.consul - :members: \ No newline at end of file + :members: From 9dabd1e77d84e50d45f62a993f236f05b60c5c1d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 14 Mar 2023 13:14:35 +0000 Subject: [PATCH 725/769] Salt no longer vendors `six` Fixes #63874 Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ff7be78..9c7eaff 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -42,26 +42,8 @@ def exec_module(self, module): return None -class SixRedirectImporter: - def find_module(self, module_name, package_path=None): - if module_name.startswith("salt.ext.six"): - return self - return None - - def load_module(self, name): - mod = importlib.import_module(name[9:]) - sys.modules[name] = mod - return mod - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - return None - - # Try our importer first -sys.meta_path = [TornadoImporter(), SixRedirectImporter()] + sys.meta_path +sys.meta_path = [TornadoImporter()] + sys.meta_path # All salt related deprecation warnings should be shown once each! From 643fb42148bdfd1835b423b56b9edb3ce21504eb Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 4 Apr 2023 11:50:44 -0700 Subject: [PATCH 726/769] Initial commit of user packages support for onedir --- src/saltext/consul/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 9c7eaff..6c192e0 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,7 +12,6 @@ ) sys.stderr.flush() - USE_VENDORED_TORNADO = True @@ -141,3 +140,11 @@ def __define_global_system_encoding_variable__(): import salt._logging # isort:skip # pylint: enable=unused-import + + +# When we are running in a 'onedir' environment, setup the path for user +# installed packages. +if hasattr(sys, "RELENV"): + sys.path = [ + str(sys.RELENV / "extras-{}.{}".format(*sys.version_info[:2])) + ] + sys.path From 522df1216a57f536d01c960fc5d028fb40350418 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Mon, 10 Apr 2023 17:44:31 -0400 Subject: [PATCH 727/769] Address review comments --- src/saltext/consul/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6c192e0..e06b8ad 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -145,6 +145,4 @@ def __define_global_system_encoding_variable__(): # When we are running in a 'onedir' environment, setup the path for user # installed packages. if hasattr(sys, "RELENV"): - sys.path = [ - str(sys.RELENV / "extras-{}.{}".format(*sys.version_info[:2])) - ] + sys.path + sys.path.insert(0, str(sys.RELENV / "extras-{}.{}".format(*sys.version_info))) From 794c40d8c41fe6eb4d423cedc5ae76c0c35e13a4 Mon Sep 17 00:00:00 2001 From: Charles McMarrow Date: Sat, 15 Apr 2023 20:19:58 -0500 Subject: [PATCH 728/769] [3006.x] rebase test labels (#64053) * changelog * add actions * add tools * typo * add some debug * pulls vs push * fire on pullr only * test port part 1 * part 2 * part 3 * part 4 * fix double run slow * clean up * update types * reg workflow * reg workflow --- tests/pytests/functional/cache/test_consul.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py index 1156b7a..3a38e49 100644 --- a/tests/pytests/functional/cache/test_consul.py +++ b/tests/pytests/functional/cache/test_consul.py @@ -67,5 +67,6 @@ def cache(minion_opts, consul_container): return cache +@pytest.mark.slow_test def test_caching(subtests, cache): run_common_cache_tests(subtests, cache) From 857ebaa04410fe88c1f47fe4df7472fc9fe4e498 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 19 Apr 2023 17:25:49 +0100 Subject: [PATCH 729/769] Make sure the file client is destroyed upon used Signed-off-by: Pedro Algarvio --- src/saltext/consul/pillar/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index 5a3f538..d324fce 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -9,7 +9,6 @@ import os import sys import traceback -import uuid import salt.channel.client import salt.ext.tornado.gen @@ -1341,6 +1340,11 @@ def destroy(self): if self._closing: return self._closing = True + if self.client: + try: + self.client.destroy() + except AttributeError: + pass # pylint: disable=W1701 def __del__(self): From 20463cd7f9038d8f2b1d38ccde9c10b26c28e807 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 3 May 2023 09:27:27 -0700 Subject: [PATCH 730/769] Remove un-needed code block --- src/saltext/consul/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e06b8ad..6649fdf 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -140,9 +140,3 @@ def __define_global_system_encoding_variable__(): import salt._logging # isort:skip # pylint: enable=unused-import - - -# When we are running in a 'onedir' environment, setup the path for user -# installed packages. -if hasattr(sys, "RELENV"): - sys.path.insert(0, str(sys.RELENV / "extras-{}.{}".format(*sys.version_info))) From 4132132221b99ace97a9700dc15651a0b1a9dd96 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 13 May 2023 21:28:32 -0700 Subject: [PATCH 731/769] Tornado 6.1 --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6649fdf..49fea82 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,7 +12,7 @@ ) sys.stderr.flush() -USE_VENDORED_TORNADO = True +USE_VENDORED_TORNADO = False class TornadoImporter: From dac8f7a09d675a648a2a6367882aa5a4c4b2859b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Thu, 18 May 2023 00:10:57 -0700 Subject: [PATCH 732/769] Fix more deltaproxy tests --- src/saltext/consul/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 49fea82..58f756d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,6 +12,7 @@ ) sys.stderr.flush() + USE_VENDORED_TORNADO = False From 1ada8ebdb57d43dbd04a3b8261e9ab17ae7be92e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 17 May 2023 04:19:07 -0700 Subject: [PATCH 733/769] Remove vendored tornado --- src/saltext/consul/__init__.py | 34 --------------------------- src/saltext/consul/pillar/__init__.py | 11 +++++---- 2 files changed, 6 insertions(+), 39 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 58f756d..485b175 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -12,40 +12,6 @@ ) sys.stderr.flush() - -USE_VENDORED_TORNADO = False - - -class TornadoImporter: - def find_module(self, module_name, package_path=None): - if USE_VENDORED_TORNADO: - if module_name.startswith("tornado"): - return self - else: - if module_name.startswith("salt.ext.tornado"): - return self - return None - - def load_module(self, name): - if USE_VENDORED_TORNADO: - mod = importlib.import_module("salt.ext.{}".format(name)) - else: - # Remove 'salt.ext.' from the module - mod = importlib.import_module(name[9:]) - sys.modules[name] = mod - return mod - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - return None - - -# Try our importer first -sys.meta_path = [TornadoImporter()] + sys.meta_path - - # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( "once", # Show once diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d324fce..bb39be5 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -10,8 +10,9 @@ import sys import traceback +import tornado.gen + import salt.channel.client -import salt.ext.tornado.gen import salt.fileclient import salt.loader import salt.minion @@ -239,7 +240,7 @@ def __init__( self._closing = False self.clean_cache = clean_cache - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def compile_pillar(self): """ Return a future which will contain the pillar data from the master @@ -277,7 +278,7 @@ def compile_pillar(self): log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! raise SaltClientError(msg) - raise salt.ext.tornado.gen.Return(ret_pillar) + raise tornado.gen.Return(ret_pillar) def destroy(self): if self._closing: @@ -1356,7 +1357,7 @@ def __del__(self): # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in # ext_pillar etc. class AsyncPillar(Pillar): - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def compile_pillar(self, ext=True): ret = super().compile_pillar(ext=ext) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) From 03c831dd846674ec165c51166520fcbf1e889ab8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 24 May 2023 07:31:49 +0100 Subject: [PATCH 734/769] Migrate `tests/unit/test_template.py` to pytest Signed-off-by: Pedro Algarvio --- tests/pytests/unit/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 43deeaa..2a30980 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -17,6 +17,7 @@ def minion_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" + opts["file_client"] = "local" return opts From 9499986e6cec4391e2d9b3119931907c689b9570 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 25 May 2023 09:17:02 +0100 Subject: [PATCH 735/769] f Signed-off-by: Pedro Algarvio --- tests/pytests/unit/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 2a30980..43deeaa 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -17,7 +17,6 @@ def minion_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" - opts["file_client"] = "local" return opts From 30f1d00fbcd85d41b1c1184c584cc656ae676b03 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 7 Jun 2023 17:13:54 +0100 Subject: [PATCH 736/769] Fix `versionadded` and/or `versionchanged` versions in docstrings Signed-off-by: Pedro Algarvio --- src/saltext/consul/cache/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 14c08cc..547e2fe 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -3,7 +3,7 @@ .. versionadded:: 2016.11.2 -.. versionchanged:: 3005.0 +.. versionchanged:: 3005 Timestamp/cache updated support added. From 1841a9388eb1a7e8f6f0714736f637670fec5fa5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 11 Jun 2023 10:03:42 +0100 Subject: [PATCH 737/769] Run pyupgrade of the modified files from the merge forward Signed-off-by: Pedro Algarvio --- src/saltext/consul/cache/consul.py | 36 ++++++++++-------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/saltext/consul/cache/consul.py b/src/saltext/consul/cache/consul.py index 547e2fe..61af6cc 100644 --- a/src/saltext/consul/cache/consul.py +++ b/src/saltext/consul/cache/consul.py @@ -119,33 +119,29 @@ def store(bank, key, data): """ Store a key value. """ - c_key = "{}/{}".format(bank, key) - tstamp_key = "{}/{}{}".format(bank, key, _tstamp_suffix) + c_key = f"{bank}/{key}" + tstamp_key = f"{bank}/{key}{_tstamp_suffix}" try: c_data = salt.payload.dumps(data) api.kv.put(c_key, c_data) api.kv.put(tstamp_key, salt.payload.dumps(int(time.time()))) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error writing the key, {}: {}".format(c_key, exc) - ) + raise SaltCacheError(f"There was an error writing the key, {c_key}: {exc}") def fetch(bank, key): """ Fetch a key value. """ - c_key = "{}/{}".format(bank, key) + c_key = f"{bank}/{key}" try: _, value = api.kv.get(c_key) if value is None: return {} return salt.payload.loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error reading the key, {}: {}".format(c_key, exc) - ) + raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") def flush(bank, key=None): @@ -156,16 +152,14 @@ def flush(bank, key=None): c_key = bank tstamp_key = None else: - c_key = "{}/{}".format(bank, key) - tstamp_key = "{}/{}{}".format(bank, key, _tstamp_suffix) + c_key = f"{bank}/{key}" + tstamp_key = f"{bank}/{key}{_tstamp_suffix}" try: if tstamp_key: api.kv.delete(tstamp_key) return api.kv.delete(c_key, recurse=key is None) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error removing the key, {}: {}".format(c_key, exc) - ) + raise SaltCacheError(f"There was an error removing the key, {c_key}: {exc}") def list_(bank): @@ -175,9 +169,7 @@ def list_(bank): try: _, keys = api.kv.get(bank + "/", keys=True, separator="/") except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - 'There was an error getting the key "{}": {}'.format(bank, exc) - ) + raise SaltCacheError(f'There was an error getting the key "{bank}": {exc}') if keys is None: keys = [] else: @@ -198,9 +190,7 @@ def contains(bank, key): c_key = "{}/{}".format(bank, key or "") _, value = api.kv.get(c_key, keys=True) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error getting the key, {}: {}".format(c_key, exc) - ) + raise SaltCacheError(f"There was an error getting the key, {c_key}: {exc}") return value is not None @@ -209,13 +199,11 @@ def updated(bank, key): Return the Unix Epoch timestamp of when the key was last updated. Return None if key is not found. """ - c_key = "{}/{}{}".format(bank, key, _tstamp_suffix) + c_key = f"{bank}/{key}{_tstamp_suffix}" try: _, value = api.kv.get(c_key) if value is None: return None return salt.payload.loads(value["Value"]) except Exception as exc: # pylint: disable=broad-except - raise SaltCacheError( - "There was an error reading the key, {}: {}".format(c_key, exc) - ) + raise SaltCacheError(f"There was an error reading the key, {c_key}: {exc}") From d8ef81932a5fa5be7c6e1662753256ee24a6b669 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 22 Feb 2023 16:46:27 +0000 Subject: [PATCH 738/769] Backport `locale.getdefaultlocale()` into Salt. It's getting removed in Py 3.13 Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 43 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 485b175..32a73d5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -39,6 +39,44 @@ ) +def __getdefaultlocale(envvars=("LC_ALL", "LC_CTYPE", "LANG", "LANGUAGE")): + """ + This function was backported from Py3.11 which started triggering a + deprecation warning about it's removal in 3.13. + """ + import locale + + try: + # check if it's supported by the _locale module + import _locale + + code, encoding = _locale._getdefaultlocale() + except (ImportError, AttributeError): + pass + else: + # make sure the code/encoding values are valid + if sys.platform == "win32" and code and code[:2] == "0x": + # map windows language identifier to language name + code = locale.windows_locale.get(int(code, 0)) + # ...add other platform-specific processing here, if + # necessary... + return code, encoding + + # fall back on POSIX behaviour + import os + + lookup = os.environ.get + for variable in envvars: + localename = lookup(variable, None) + if localename: + if variable == "LANGUAGE": + localename = localename.split(":")[0] + break + else: + localename = "C" + return locale._parse_localename(localename) + + def __define_global_system_encoding_variable__(): import sys @@ -57,17 +95,14 @@ def __define_global_system_encoding_variable__(): # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong # encoding - import locale try: - encoding = locale.getdefaultlocale()[-1] + encoding = __getdefaultlocale()[-1] except ValueError: # A bad locale setting was most likely found: # https://github.com/saltstack/salt/issues/26063 pass - # This is now garbage collectable - del locale if not encoding: # This is most likely ascii which is not the best but we were # unable to find a better encoding. If this fails, we fall all From dae869c9c7d672a85103054a75ed8745d5453cad Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Jun 2023 11:04:46 +0100 Subject: [PATCH 739/769] Stop using the deprecated `locale.getdefaultlocale()` function Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 6649fdf..30880b1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -93,14 +93,14 @@ def __define_global_system_encoding_variable__(): import locale try: - encoding = locale.getdefaultlocale()[-1] - except ValueError: - # A bad locale setting was most likely found: - # https://github.com/saltstack/salt/issues/26063 - pass + encoding = locale.getencoding() + except AttributeError: + # Python < 3.11 + encoding = locale.getpreferredencoding(do_setlocale=True) # This is now garbage collectable del locale + if not encoding: # This is most likely ascii which is not the best but we were # unable to find a better encoding. If this fails, we fall all From c447fe7c5632354f952e33e283911ed2114e8afc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Jun 2023 11:57:23 +0100 Subject: [PATCH 740/769] Stop using the deprecated `locale.getdefaultlocale()` function Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 51 ++++++---------------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 32a73d5..4407a7d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -39,44 +39,6 @@ ) -def __getdefaultlocale(envvars=("LC_ALL", "LC_CTYPE", "LANG", "LANGUAGE")): - """ - This function was backported from Py3.11 which started triggering a - deprecation warning about it's removal in 3.13. - """ - import locale - - try: - # check if it's supported by the _locale module - import _locale - - code, encoding = _locale._getdefaultlocale() - except (ImportError, AttributeError): - pass - else: - # make sure the code/encoding values are valid - if sys.platform == "win32" and code and code[:2] == "0x": - # map windows language identifier to language name - code = locale.windows_locale.get(int(code, 0)) - # ...add other platform-specific processing here, if - # necessary... - return code, encoding - - # fall back on POSIX behaviour - import os - - lookup = os.environ.get - for variable in envvars: - localename = lookup(variable, None) - if localename: - if variable == "LANGUAGE": - localename = localename.split(":")[0] - break - else: - localename = "C" - return locale._parse_localename(localename) - - def __define_global_system_encoding_variable__(): import sys @@ -95,13 +57,16 @@ def __define_global_system_encoding_variable__(): # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong # encoding + import locale try: - encoding = __getdefaultlocale()[-1] - except ValueError: - # A bad locale setting was most likely found: - # https://github.com/saltstack/salt/issues/26063 - pass + encoding = locale.getencoding() + except AttributeError: + # Python < 3.11 + encoding = locale.getpreferredencoding(do_setlocale=True) + + # This is now garbage collectable + del locale if not encoding: # This is most likely ascii which is not the best but we were From 0a6f7c28cb453d834078f78562f48b9db57b7cf5 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 19 Jun 2023 16:14:11 -0700 Subject: [PATCH 741/769] Add import hook for nacl --- src/saltext/consul/__init__.py | 54 ++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 30880b1..d843a5c 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -5,6 +5,8 @@ import importlib import sys import warnings +import logging +log = logging.getLogger() if sys.version_info < (3,): sys.stderr.write( @@ -16,6 +18,7 @@ class TornadoImporter: + def find_module(self, module_name, package_path=None): if USE_VENDORED_TORNADO: if module_name.startswith("tornado"): @@ -25,24 +28,63 @@ def find_module(self, module_name, package_path=None): return self return None - def load_module(self, name): + def create_module(self, spec): if USE_VENDORED_TORNADO: - mod = importlib.import_module("salt.ext.{}".format(name)) + mod = importlib.import_module("salt.ext.{}".format(sepc.name)) else: # Remove 'salt.ext.' from the module - mod = importlib.import_module(name[9:]) + mod = importlib.import_module(sepc.name[9:]) sys.modules[name] = mod return mod + def exec_module(self, module): + log.error("exec_module %r", module) + return None + + +class NaclImporter: + """ + Import hook to force PyNaCl to perform dlopen on libsodium with the + RTLD_DEEPBIND flag. This is to work around an issue where pyzmq does a dlopen + with RTLD_GLOBAL which the causes calls to libsodium to resolve to + tweetnacl when it's been bundled with pyzmq. + + See: https://github.com/zeromq/pyzmq/issues/1878 + """ + loading = False + + def find_module(self, module_name, package_path=None): + if not NaclImporter.loading and module_name == "nacl": + NaclImporter.loading = True + return self + return None + def create_module(self, spec): - return self.load_module(spec.name) + dlopen = hasattr(sys, "getdlopenflags") + if dlopen: + dlflags = sys.getdlopenflags() + # Use RTDL_DEEPBIND in case pyzmq was compiled with ZMQ_USE_TWEETNACL. This is + # needed because pyzmq imports libzmq with RTLD_GLOBAL. + if hasattr(os, "RTLD_DEEPBIND"): + flags = os.RTLD_DEEPBIND | dlflags + else: + flags = dlflags + sys.setdlopenflags(dlflags) + try: + mod = importlib.import_module(spec.name) + finally: + if dlopen: + sys.setdlopenflags(dlflags) + NaclImporter.loading = False + sys.modules[name] = mod + return mod def exec_module(self, module): + log.error("exec_module %r", module) return None - # Try our importer first -sys.meta_path = [TornadoImporter()] + sys.meta_path +sys.meta_path = [TornadoImporter(), NaclImporter] + sys.meta_path # All salt related deprecation warnings should be shown once each! From 6e22e25ea32f93eb0be699a883a392f14825775e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 20 Jun 2023 08:42:39 -0700 Subject: [PATCH 742/769] Add import hook for nacl --- src/saltext/consul/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index d843a5c..8968a70 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -3,9 +3,11 @@ """ import importlib +import logging +import os import sys import warnings -import logging + log = logging.getLogger() if sys.version_info < (3,): @@ -18,7 +20,6 @@ class TornadoImporter: - def find_module(self, module_name, package_path=None): if USE_VENDORED_TORNADO: if module_name.startswith("tornado"): @@ -30,11 +31,11 @@ def find_module(self, module_name, package_path=None): def create_module(self, spec): if USE_VENDORED_TORNADO: - mod = importlib.import_module("salt.ext.{}".format(sepc.name)) + mod = importlib.import_module("salt.ext.{}".format(spec.name)) else: # Remove 'salt.ext.' from the module - mod = importlib.import_module(sepc.name[9:]) - sys.modules[name] = mod + mod = importlib.import_module(spec.name[9:]) + sys.modules[spec.name] = mod return mod def exec_module(self, module): @@ -51,6 +52,7 @@ class NaclImporter: See: https://github.com/zeromq/pyzmq/issues/1878 """ + loading = False def find_module(self, module_name, package_path=None): @@ -76,13 +78,14 @@ def create_module(self, spec): if dlopen: sys.setdlopenflags(dlflags) NaclImporter.loading = False - sys.modules[name] = mod + sys.modules[spec.name] = mod return mod def exec_module(self, module): log.error("exec_module %r", module) return None + # Try our importer first sys.meta_path = [TornadoImporter(), NaclImporter] + sys.meta_path From cfb325e1585c8ce0de1c670d1988d7bb98e2afb8 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 20 Jun 2023 08:53:34 -0700 Subject: [PATCH 743/769] Fix typo --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 8968a70..651145d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -71,7 +71,7 @@ def create_module(self, spec): flags = os.RTLD_DEEPBIND | dlflags else: flags = dlflags - sys.setdlopenflags(dlflags) + sys.setdlopenflags(flags) try: mod = importlib.import_module(spec.name) finally: From e235e79bc963d85a4e49b7ac50e00f886d4a3408 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 20 Jun 2023 13:55:59 -0700 Subject: [PATCH 744/769] Fix warts in patch and linter errors --- src/saltext/consul/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 651145d..29b290d 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -56,7 +56,8 @@ class NaclImporter: loading = False def find_module(self, module_name, package_path=None): - if not NaclImporter.loading and module_name == "nacl": + print(f"{self} {module_name}") + if not NaclImporter.loading and module_name.startswith("nacl"): NaclImporter.loading = True return self return None @@ -87,7 +88,7 @@ def exec_module(self, module): # Try our importer first -sys.meta_path = [TornadoImporter(), NaclImporter] + sys.meta_path +sys.meta_path = [TornadoImporter(), NaclImporter()] + sys.meta_path # All salt related deprecation warnings should be shown once each! From e65e6b372002108c1fdf979ec6880f7d3cc560ee Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 21 Jun 2023 09:10:12 -0700 Subject: [PATCH 745/769] Remove debug print --- src/saltext/consul/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 29b290d..14c7235 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -56,7 +56,6 @@ class NaclImporter: loading = False def find_module(self, module_name, package_path=None): - print(f"{self} {module_name}") if not NaclImporter.loading and module_name.startswith("nacl"): NaclImporter.loading = True return self From e7fd16b2212dacca8551ffc74b96048374e32a14 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 28 Jun 2023 10:32:28 -0600 Subject: [PATCH 746/769] Update salt/__init__.py Co-authored-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 14c7235..f8b426a 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -47,7 +47,7 @@ class NaclImporter: """ Import hook to force PyNaCl to perform dlopen on libsodium with the RTLD_DEEPBIND flag. This is to work around an issue where pyzmq does a dlopen - with RTLD_GLOBAL which the causes calls to libsodium to resolve to + with RTLD_GLOBAL which then causes calls to libsodium to resolve to tweetnacl when it's been bundled with pyzmq. See: https://github.com/zeromq/pyzmq/issues/1878 From caacd270fd13056b6d9fc23e723cacd516686e99 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 30 Jun 2023 16:42:48 -0600 Subject: [PATCH 747/769] Updated per reviewer comments --- src/saltext/consul/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f8b426a..7542ef8 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -38,10 +38,6 @@ def create_module(self, spec): sys.modules[spec.name] = mod return mod - def exec_module(self, module): - log.error("exec_module %r", module) - return None - class NaclImporter: """ @@ -81,10 +77,6 @@ def create_module(self, spec): sys.modules[spec.name] = mod return mod - def exec_module(self, module): - log.error("exec_module %r", module) - return None - # Try our importer first sys.meta_path = [TornadoImporter(), NaclImporter()] + sys.meta_path From 8a22ddca1302f95f8c69f2227176e0821ffdc6fe Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Fri, 14 Jul 2023 16:44:06 -0600 Subject: [PATCH 748/769] Restore exec_module to TornadoImporter and NaclImporter --- src/saltext/consul/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 7542ef8..edf7ff1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -38,6 +38,9 @@ def create_module(self, spec): sys.modules[spec.name] = mod return mod + def exec_module(self, module): + return None + class NaclImporter: """ @@ -77,6 +80,9 @@ def create_module(self, spec): sys.modules[spec.name] = mod return mod + def exec_module(self, module): + return None + # Try our importer first sys.meta_path = [TornadoImporter(), NaclImporter()] + sys.meta_path From 260dcc3971a6a7ab537c437ea9e1d4c3d9a53175 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Mon, 17 Jul 2023 11:03:59 -0600 Subject: [PATCH 749/769] Removed debug logging and updated requirements per reviewer suggestions --- src/saltext/consul/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index edf7ff1..f75da5e 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -3,13 +3,10 @@ """ import importlib -import logging import os import sys import warnings -log = logging.getLogger() - if sys.version_info < (3,): sys.stderr.write( "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" From 50ce905fef0af7c855b3b770ee33764cc48fec3d Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 31 Jul 2023 17:28:07 -0700 Subject: [PATCH 750/769] Reasonable failures when pillars timeout --- src/saltext/consul/pillar/__init__.py | 33 +++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/saltext/consul/pillar/__init__.py b/src/saltext/consul/pillar/__init__.py index d324fce..d047f36 100644 --- a/src/saltext/consul/pillar/__init__.py +++ b/src/saltext/consul/pillar/__init__.py @@ -8,6 +8,7 @@ import logging import os import sys +import time import traceback import salt.channel.client @@ -259,6 +260,7 @@ def compile_pillar(self): if self.ext: load["ext"] = self.ext try: + start = time.monotonic() ret_pillar = yield self.channel.crypted_transfer_decode_dictentry( load, dictkey="pillar", @@ -266,6 +268,10 @@ def compile_pillar(self): except salt.crypt.AuthenticationError as exc: log.error(exc.message) raise SaltClientError("Exception getting pillar.") + except salt.exceptions.SaltReqTimeoutError: + raise SaltClientError( + f"Pillar timed out after {int(time.monotonic() - start)} seconds" + ) except Exception: # pylint: disable=broad-except log.exception("Exception getting pillar:") raise SaltClientError("Exception getting pillar.") @@ -350,10 +356,23 @@ def compile_pillar(self): } if self.ext: load["ext"] = self.ext - ret_pillar = self.channel.crypted_transfer_decode_dictentry( - load, - dictkey="pillar", - ) + + try: + start = time.monotonic() + ret_pillar = self.channel.crypted_transfer_decode_dictentry( + load, + dictkey="pillar", + ) + except salt.crypt.AuthenticationError as exc: + log.error(exc.message) + raise SaltClientError("Exception getting pillar.") + except salt.exceptions.SaltReqTimeoutError: + raise SaltClientError( + f"Pillar timed out after {int(time.monotonic() - start)} seconds" + ) + except Exception: # pylint: disable=broad-except + log.exception("Exception getting pillar:") + raise SaltClientError("Exception getting pillar.") if not isinstance(ret_pillar, dict): log.error( @@ -925,7 +944,7 @@ def render_pstate(self, sls, saltenv, mods, defaults=None): saltenv, sls, _pillar_rend=True, - **defaults + **defaults, ) except Exception as exc: # pylint: disable=broad-except msg = "Rendering SLS '{}' failed, render error:\n{}".format(sls, exc) @@ -1104,7 +1123,7 @@ def _external_pillar_data(self, pillar, val, key): self.minion_id, pillar, extra_minion_data=self.extra_minion_data, - **val + **val, ) else: ext = self.ext_pillars[key](self.minion_id, pillar, **val) @@ -1114,7 +1133,7 @@ def _external_pillar_data(self, pillar, val, key): self.minion_id, pillar, *val, - extra_minion_data=self.extra_minion_data + extra_minion_data=self.extra_minion_data, ) else: ext = self.ext_pillars[key](self.minion_id, pillar, *val) From 2877476b6df091c937f32dbfd5f92b5e6a0de40d Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Wed, 14 Jun 2023 11:38:52 -0600 Subject: [PATCH 751/769] ensure we unset ONEDIR env for unit/functional tests --- tests/pytests/functional/conftest.py | 18 +++++++++++++++++- tests/pytests/unit/conftest.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index 835464e..b2e6607 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,4 +1,5 @@ import logging +import os import shutil import pytest @@ -7,6 +8,21 @@ log = logging.getLogger(__name__) +@pytest.fixture(scope="package", autouse=True) +def onedir_env(): + """ + Functional tests cannot currently test the + onedir artifact. This will need to be removed + when we do add onedir support for functional tests. + """ + if os.environ.get("ONEDIR_TESTRUN", "0") == "1": + try: + os.environ["ONEDIR_TESTRUN"] = "0" + yield + finally: + os.environ["ONEDIR_TESTRUN"] = "1" + + @pytest.fixture(scope="package") def minion_id(): return "func-tests-minion-opts" @@ -127,7 +143,7 @@ def master_opts( @pytest.fixture(scope="module") def loaders(minion_opts): - return Loaders(minion_opts, loaded_base_name="{}.loaded".format(__name__)) + return Loaders(minion_opts, loaded_base_name=f"{__name__}.loaded") @pytest.fixture(autouse=True) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 43deeaa..24d6e7e 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -1,8 +1,25 @@ +import os + import pytest import salt.config +@pytest.fixture(scope="package", autouse=True) +def onedir_env(): + """ + Unit tests cannot currently test the + onedir artifact. This will need to be removed + when we do add onedir support for functional tests. + """ + if os.environ.get("ONEDIR_TESTRUN", "0") == "1": + try: + os.environ["ONEDIR_TESTRUN"] = "0" + yield + finally: + os.environ["ONEDIR_TESTRUN"] = "1" + + @pytest.fixture def minion_opts(tmp_path): """ From 3d0db494e4fbba0b3f897f64289a5fda9cde1080 Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Tue, 20 Jun 2023 08:40:21 -0600 Subject: [PATCH 752/769] Run onedir pytest fixture only on module tests --- tests/pytests/functional/conftest.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/pytests/functional/conftest.py b/tests/pytests/functional/conftest.py index b2e6607..2fb2246 100644 --- a/tests/pytests/functional/conftest.py +++ b/tests/pytests/functional/conftest.py @@ -1,5 +1,4 @@ import logging -import os import shutil import pytest @@ -8,21 +7,6 @@ log = logging.getLogger(__name__) -@pytest.fixture(scope="package", autouse=True) -def onedir_env(): - """ - Functional tests cannot currently test the - onedir artifact. This will need to be removed - when we do add onedir support for functional tests. - """ - if os.environ.get("ONEDIR_TESTRUN", "0") == "1": - try: - os.environ["ONEDIR_TESTRUN"] = "0" - yield - finally: - os.environ["ONEDIR_TESTRUN"] = "1" - - @pytest.fixture(scope="package") def minion_id(): return "func-tests-minion-opts" From c3879c19cc2fd552f7003c370e268239d52a91cc Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Thu, 20 Jul 2023 13:15:28 -0600 Subject: [PATCH 753/769] Move _pkg.txt into salt directory --- tests/pytests/unit/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 24d6e7e..9c88357 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -6,7 +6,7 @@ @pytest.fixture(scope="package", autouse=True) -def onedir_env(): +def _onedir_env(): """ Unit tests cannot currently test the onedir artifact. This will need to be removed @@ -18,6 +18,8 @@ def onedir_env(): yield finally: os.environ["ONEDIR_TESTRUN"] = "1" + else: + yield @pytest.fixture From d5bf1042cdeeca894cb5aa57faa88d16044b630d Mon Sep 17 00:00:00 2001 From: Megan Wilhite Date: Wed, 2 Aug 2023 11:45:02 -0600 Subject: [PATCH 754/769] Remove onedir pytest fixture workaround --- tests/pytests/unit/conftest.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 9c88357..43deeaa 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -1,27 +1,8 @@ -import os - import pytest import salt.config -@pytest.fixture(scope="package", autouse=True) -def _onedir_env(): - """ - Unit tests cannot currently test the - onedir artifact. This will need to be removed - when we do add onedir support for functional tests. - """ - if os.environ.get("ONEDIR_TESTRUN", "0") == "1": - try: - os.environ["ONEDIR_TESTRUN"] = "0" - yield - finally: - os.environ["ONEDIR_TESTRUN"] = "1" - else: - yield - - @pytest.fixture def minion_opts(tmp_path): """ From aa847c0b5119b79f9a121fa14f7a76359d6541e1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 27 Jun 2023 16:34:50 -0700 Subject: [PATCH 755/769] More test fixes --- src/saltext/consul/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index ed6ce93..e73e7bd 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,11 +2,15 @@ Salt package """ +import asyncio import importlib import os import sys import warnings +if sys.platform.startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + if sys.version_info < (3,): sys.stderr.write( "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" @@ -90,6 +94,9 @@ def exec_module(self, module): def __define_global_system_encoding_variable__(): import sys + print("define global system encoding") + sys.stdout.flush() + # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None From f2343fb2935765ea41ec1661ede8237c4aed15fa Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 27 Jun 2023 22:24:13 -0700 Subject: [PATCH 756/769] More cleanup --- src/saltext/consul/__init__.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index e73e7bd..52c1ad1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -4,6 +4,7 @@ import asyncio import importlib +import locale import os import sys import warnings @@ -92,11 +93,6 @@ def exec_module(self, module): def __define_global_system_encoding_variable__(): - import sys - - print("define global system encoding") - sys.stdout.flush() - # This is the most trustworthy source of the system encoding, though, if # salt is being imported after being daemonized, this information is lost # and reset to None @@ -112,17 +108,12 @@ def __define_global_system_encoding_variable__(): # If the system is properly configured this should return a valid # encoding. MS Windows has problems with this and reports the wrong # encoding - import locale - try: encoding = locale.getencoding() except AttributeError: # Python < 3.11 encoding = locale.getpreferredencoding(do_setlocale=True) - # This is now garbage collectable - del locale - if not encoding: # This is most likely ascii which is not the best but we were # unable to find a better encoding. If this fails, we fall all @@ -146,7 +137,6 @@ def __define_global_system_encoding_variable__(): setattr(builtins, "__salt_system_encoding__", encoding) # This is now garbage collectable - del sys del builtins del encoding From 7442e8918f78c0eb761d341834b992d7f417b92f Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 31 Jul 2023 14:13:38 -0700 Subject: [PATCH 757/769] Use fixtures when already present --- tests/pytests/unit/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 43deeaa..0f8478f 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -26,7 +26,7 @@ def master_opts(tmp_path): Default master configuration with relative temporary paths to not require root permissions. """ root_dir = tmp_path / "master" - opts = salt.config.DEFAULT_MASTER_OPTS.copy() + opts = salt.config.master_config(None) opts["__role"] = "master" opts["root_dir"] = str(root_dir) for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): From bb8f19b09704576b5a1fb465c22bd68db9178e5b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 26 Aug 2023 01:22:29 -0700 Subject: [PATCH 758/769] Syndic's async request channel is actually async --- tests/pytests/unit/conftest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 43deeaa..443f0b6 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -35,3 +35,21 @@ def master_opts(tmp_path): opts[name] = str(dirpath) opts["log_file"] = "logs/master.log" return opts + + +@pytest.fixture +def syndic_opts(tmp_path): + """ + Default master configuration with relative temporary paths to not require root permissions. + """ + root_dir = tmp_path / "syndic" + opts = salt.config.DEFAULT_MINION_OPTS.copy() + opts["syndic_master"] = "127.0.0.1" + opts["__role"] = "minion" + opts["root_dir"] = str(root_dir) + for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): + dirpath = root_dir / name + dirpath.mkdir(parents=True) + opts[name] = str(dirpath) + opts["log_file"] = "logs/syndic.log" + return opts From 53f44e3f50c7b1aff36167988a0b8ba5cb85925d Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Aug 2023 14:41:15 -0700 Subject: [PATCH 759/769] Ignore warnings to see if packages work --- src/saltext/consul/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index f75da5e..cbf6281 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -111,6 +111,11 @@ def exec_module(self, module): module="_distutils_hack", ) +warnings.filterwarnings( + "ignore", + message="invalid escape sequence.*", + category=DeprecationWarning, +) def __define_global_system_encoding_variable__(): import sys From 1139476cf44124405370c895910f09bed7fa1e83 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 4 Sep 2023 15:24:16 -0700 Subject: [PATCH 760/769] Bump relenv version to 0.13.9 --- src/saltext/consul/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cbf6281..cc35272 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -13,6 +13,7 @@ ) sys.stderr.flush() + USE_VENDORED_TORNADO = True @@ -117,6 +118,7 @@ def exec_module(self, module): category=DeprecationWarning, ) + def __define_global_system_encoding_variable__(): import sys From 49f2f632fec80b444962f4994761a73abb8f2d63 Mon Sep 17 00:00:00 2001 From: cmcmarrow Date: Wed, 6 Sep 2023 17:27:06 -0500 Subject: [PATCH 761/769] port minion_opts --- tests/pytests/unit/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/pytests/unit/conftest.py diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py new file mode 100644 index 0000000..8b92547 --- /dev/null +++ b/tests/pytests/unit/conftest.py @@ -0,0 +1,19 @@ +import pytest +import salt.config + + +@pytest.fixture +def minion_opts(tmp_path): + """ + Default minion configuration with relative temporary paths to not require root permissions. + """ + root_dir = tmp_path / "minion" + opts = salt.config.DEFAULT_MINION_OPTS.copy() + opts["__role"] = "minion" + opts["root_dir"] = str(root_dir) + for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): + dirpath = root_dir / name + dirpath.mkdir(parents=True) + opts[name] = str(dirpath) + opts["log_file"] = "logs/minion.log" + return opts From e54fb90bdcff8939a8f83605c0c14221ba7e821d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 10 Oct 2023 20:45:56 +0100 Subject: [PATCH 762/769] Some tests require `TCPPubClient` to be mocked. This prevents the tests from hanging on Windows, and slowing down on Linux. Signed-off-by: Pedro Algarvio --- tests/pytests/unit/conftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index cf839ba..5283510 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -1,6 +1,10 @@ +import asyncio + import pytest import salt.config +import salt.transport.tcp +from tests.support.mock import MagicMock, patch @pytest.fixture @@ -53,3 +57,14 @@ def syndic_opts(tmp_path): opts[name] = str(dirpath) opts["log_file"] = "logs/syndic.log" return opts + + +@pytest.fixture +def mocked_tcp_pub_client(): + transport = MagicMock(spec=salt.transport.tcp.TCPPubClient) + transport.connect = MagicMock() + future = asyncio.Future() + transport.connect.return_value = future + future.set_result(True) + with patch("salt.transport.tcp.TCPPubClient", transport): + yield From c48d18a185b6df6ba70956256021588d8ca0edf5 Mon Sep 17 00:00:00 2001 From: MKLeb Date: Thu, 12 Oct 2023 20:09:44 -0400 Subject: [PATCH 763/769] Exclude some un-executed lines in `salt/__init__.py` from coverage, they aren't ever going to run. We may even be able to remove them --- src/saltext/consul/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index cc35272..a931411 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -7,7 +7,7 @@ import sys import warnings -if sys.version_info < (3,): +if sys.version_info < (3,): # pragma: no cover sys.stderr.write( "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" ) @@ -22,7 +22,7 @@ def find_module(self, module_name, package_path=None): if USE_VENDORED_TORNADO: if module_name.startswith("tornado"): return self - else: + else: # pragma: no cover if module_name.startswith("salt.ext.tornado"): return self return None @@ -30,7 +30,7 @@ def find_module(self, module_name, package_path=None): def create_module(self, spec): if USE_VENDORED_TORNADO: mod = importlib.import_module("salt.ext.{}".format(spec.name)) - else: + else: # pragma: no cover # Remove 'salt.ext.' from the module mod = importlib.import_module(spec.name[9:]) sys.modules[spec.name] = mod From 3ea39d48c34743b9cceaaed79f5f1527c5f5ce70 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 27 Oct 2023 16:29:05 +0100 Subject: [PATCH 764/769] Skip tests which can't run, or even pass on FIPS enabled platforms Signed-off-by: Pedro Algarvio --- tests/pytests/functional/cache/test_consul.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py index 3a38e49..0a42913 100644 --- a/tests/pytests/functional/cache/test_consul.py +++ b/tests/pytests/functional/cache/test_consul.py @@ -14,6 +14,7 @@ log = logging.getLogger(__name__) pytestmark = [ + pytest.mark.skip_on_fips_enabled_platform, pytest.mark.slow_test, pytest.mark.skip_if_binaries_missing("dockerd"), ] From 0e3e5172f76bf780c7e827d3b74e7104e0f00517 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 26 Nov 2023 18:06:09 +0000 Subject: [PATCH 765/769] Don't change the default asyncio loop policy Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index be81088..116d7b5 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,16 +2,12 @@ Salt package """ -import asyncio import importlib import locale import os import sys import warnings -if sys.platform.startswith("win"): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - if sys.version_info < (3,): # pragma: no cover sys.stderr.write( "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" From 8feccecaf520114a6cdc480e59e2a9501f6c20df Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 22 Nov 2023 18:30:22 +0000 Subject: [PATCH 766/769] Ignore some `pkg_resources` related `DeprecationWarnings` There's nothing Salt can do, they are triggered by it's dependencies. Signed-off-by: Pedro Algarvio --- src/saltext/consul/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index a931411..a7c32e1 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -118,6 +118,17 @@ def exec_module(self, module): category=DeprecationWarning, ) +warnings.filterwarnings( + "ignore", + "Deprecated call to `pkg_resources.declare_namespace.*", + category=DeprecationWarning, +) +warnings.filterwarnings( + "ignore", + ".*pkg_resources is deprecated as an API.*", + category=DeprecationWarning, +) + def __define_global_system_encoding_variable__(): import sys From 3546bef43a6350a874ea25a6c3cd12f615ef61be Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 1 Dec 2023 01:16:03 -0700 Subject: [PATCH 767/769] Add master_uri to minion_opts for tests --- tests/pytests/unit/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 443f0b6..587fc43 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -12,6 +12,9 @@ def minion_opts(tmp_path): opts = salt.config.DEFAULT_MINION_OPTS.copy() opts["__role"] = "minion" opts["root_dir"] = str(root_dir) + opts["master_uri"] = "tcp://{ip}:{port}".format( + ip="127.0.0.1", port=opts["master_port"] + ) for name in ("cachedir", "pki_dir", "sock_dir", "conf_dir"): dirpath = root_dir / name dirpath.mkdir(parents=True) From 6007805cd5474dc4fcd046e91620b6cb35a26e38 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 7 Dec 2023 17:05:37 +0000 Subject: [PATCH 768/769] Revert "Don't change the default asyncio loop policy" This reverts commit 0e3e5172f76bf780c7e827d3b74e7104e0f00517. --- src/saltext/consul/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/saltext/consul/__init__.py b/src/saltext/consul/__init__.py index 5a89513..87b58d2 100644 --- a/src/saltext/consul/__init__.py +++ b/src/saltext/consul/__init__.py @@ -2,12 +2,16 @@ Salt package """ +import asyncio import importlib import locale import os import sys import warnings +if sys.platform.startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + if sys.version_info < (3,): # pragma: no cover sys.stderr.write( "\n\nAfter the Sodium release, 3001, Salt no longer supports Python 2. Exiting.\n\n" From 2e8d01077f4d67562542e38055772c5dd29cfdb0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 13 Dec 2023 05:28:12 +0000 Subject: [PATCH 769/769] Set the right path for the config file Signed-off-by: Pedro Algarvio --- tests/pytests/unit/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 587fc43..c7152f3 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -1,3 +1,5 @@ +import os + import pytest import salt.config @@ -20,6 +22,7 @@ def minion_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/minion.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "minion") return opts @@ -37,6 +40,7 @@ def master_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/master.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "master") return opts @@ -55,4 +59,5 @@ def syndic_opts(tmp_path): dirpath.mkdir(parents=True) opts[name] = str(dirpath) opts["log_file"] = "logs/syndic.log" + opts["conf_file"] = os.path.join(opts["conf_dir"], "syndic") return opts