From 16cbc1d4a1eca7f061591d561b9c61e44aea79fc Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:29:36 +0100 Subject: [PATCH 1/9] test(inspec): verify the proper loading of parameters - the test file must include the `map_jinja:sources` - the `bar` role provide two parameters but the lookup override the `winner` - adapt rubocop since `config_spec.rb` is more that 36 lines --- .rubocop.yml | 5 ++++- TEMPLATE/parameters/roles/bar.yaml | 15 +++++++++++++++ test/integration/default/controls/config_spec.rb | 8 ++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 TEMPLATE/parameters/roles/bar.yaml diff --git a/.rubocop.yml b/.rubocop.yml index 3b6c0b8b..4e406eb7 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,5 +10,8 @@ Layout/LineLength: # Any offenses that should be fixed, e.g. collected via. `rubocop --auto-gen-config` # diff --git a/TEMPLATE/parameters/roles/bar.yaml b/TEMPLATE/parameters/roles/bar.yaml new file mode 100644 index 00000000..57402317 --- /dev/null +++ b/TEMPLATE/parameters/roles/bar.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Variables specific to the role "bar" looked up with salt['config.get']('roles'). +# +# This is only for demonstration and testing purpose: +# - we test the presence of parameter "added_in_role_bar" and the +# corresponding value +# - we make test that the "winner" parameter is properly overwritten +# by pillar +--- +strategy: 'overwrite' +values: + added_in_role_bar: 'bar_role_value' + winner: 'role bar' diff --git a/test/integration/default/controls/config_spec.rb b/test/integration/default/controls/config_spec.rb index 68b5b4bc..d0ee5bd2 100644 --- a/test/integration/default/controls/config_spec.rb +++ b/test/integration/default/controls/config_spec.rb @@ -32,5 +32,13 @@ its('content') { should include '"arch": "amd64"' } its('content') { should include '"winner": "pillar"}' } its('content') { should include 'winner of the merge: pillar' } + its('content') do + should include( + '"map_jinja": {"sources": ["osarch", "os_family", "os", "osfinger", ' \ + '"roles", "config_get_lookup", "config_get", "id", ' \ + '"any/path/can/be/used/here.yaml"]}' + ) + end + its('content') { should include '"added_in_role_bar": "bar_role_value"' } end end From 707822cfa2f97546131200b2b459bf8233e1c560 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:35:04 +0100 Subject: [PATCH 2/9] feat(map.jinja): load configuration values from configurable sources Lookup the configuration for `map.jinja` from: 1. builtin default 1. `osarch` grain 2. `os_family` grain 3. `os` grain 4. `osfinger` grain 5. `lookup` table retrived by `config.get` 6. configuration retrived by `config.get` 7. `id` grain 2. `defaults.yaml`: optionally define a formula specific `map_jinja:sources` 3. global configuration lookup `map_jinja:sources` 4. formula specific `:map_jinja:sources` The hard coded default is backward compatible and add the minion id at the end of the list. If an entry does not match a `salt['config.get']` parameter, it's used as a literal file path (with `.yaml` added if it's missing). Each YAML file are formatted with the following top level keys: - `values`: contains the the parameter values - `strategy` (optional): define the merge strategy for the function `salt.slsutils.merge` (aggregate, list, overwrite, recurse, smart). The default is `smart`. - `merge_lists` (optional): boolean to merge lists or overwrite them It's possile to configure `config.get` when looking up the formula configuration. You need to define a subkey `strategy` with the following differences from YAML files: - `strategy` can only be one of `None`, `overwrite` and `recurse`. The default is `None`. - if you use `salt-ssh`, this merge strategy is skipped and and error log message is emitted. The parameters values are merged in the following order: 1. initialize default values from `defaults.yaml` 2. merge the values from each source defined by the ordered list `map_jinja:sources` --- TEMPLATE/map.jinja | 153 ++++++++++++++++++++++++++++++++------------- 1 file changed, 110 insertions(+), 43 deletions(-) diff --git a/TEMPLATE/map.jinja b/TEMPLATE/map.jinja index 1b8b1c67..9c5888e4 100644 --- a/TEMPLATE/map.jinja +++ b/TEMPLATE/map.jinja @@ -3,51 +3,118 @@ {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} -{#- Start imports as #} -{%- import_yaml tplroot ~ "/defaults.yaml" as default_settings %} -{%- import_yaml tplroot ~ "/osarchmap.yaml" as osarchmap %} -{%- import_yaml tplroot ~ "/osfamilymap.yaml" as osfamilymap %} -{%- import_yaml tplroot ~ "/osmap.yaml" as osmap %} -{%- import_yaml tplroot ~ "/osfingermap.yaml" as osfingermap %} - -{#- Retrieve the config dict only once #} -{%- set _config = salt['config.get'](tplroot, default={}) %} - -{%- set defaults = salt['grains.filter_by']( - default_settings, - default=tplroot, - merge=salt['grains.filter_by']( - osarchmap, - grain='osarch', - merge=salt['grains.filter_by']( - osfamilymap, - grain='os_family', - merge=salt['grains.filter_by']( - osmap, - grain='os', - merge=salt['grains.filter_by']( - osfingermap, - grain='osfinger', - merge=salt['grains.filter_by']( - _config, - default='lookup' - ) - ) - ) - ) - ) - ) -%} - -{%- set config = salt['grains.filter_by']( - {'defaults': defaults}, - default='defaults', - merge=_config - ) -%} +{%- from tplroot ~ "/libsaltcli.jinja" import cli with context %} + +{#- Where to lookup parameters source files #} +{%- set map_sources_dir = tplroot | path_join('parameters') %} + +{#- Load defaults first to allow per formula default map.jinja configuration #} +{%- set _defaults_filename = map_sources_dir | path_join('defaults.yaml') %} +{%- do salt['log.debug']('map.jinja: initialise parameters from ' ~ _defaults_filename ) %} +{%- import_yaml _defaults_filename as default_settings %} + +{#- List of sources to lookup for parameters #} +{%- do salt['log.debug']("map.jinja: lookup 'map_jinja' configuration sources") %} +{#- Fallback to previously used grains plus minion `id` #} +{%- set map_sources = ['osarch', 'os_family', 'os', 'osfinger', 'config_get_lookup', 'config_get', 'id'] %} +{#- Configure map.jinja from defaults.yaml #} +{%- set map_sources = default_settings | traverse('values:map_jinja:sources', map_sources) %} + +{#- Lookup global sources #} +{%- set map_sources = salt['config.get']('map_jinja:sources', map_sources) %} +{#- Lookup per formula sources #} +{%- set map_sources = salt['config.get'](tplroot ~ ':map_jinja:sources', map_sources) %} + +{%- do salt['log.debug']('map.jinja: load parameters with sources from ' ~ map_sources) %} + + +{#- Work around assignment inside for loop #} +{#- load configuration values used in `config.get` merging strategies #} +{%- set _config = {'stack': default_settings.get('values', {}), + 'merge_strategy': salt['config.get'](tplroot ~ ':strategy', None), + 'merge_lists': salt['config.get'](tplroot ~ ':merge_lists', False) + } %} + +{#- the `config.get` merge option only works for `minion` or `local` salt command types #} +{%- if cli in ['minion', 'local'] %} + {%- do _config.update({'merge_opt': {'merge': _config['merge_strategy']}, + 'merge_msg': ", merge: strategy='" ~ _config['merge_strategy'] ~ "'"}) %} +{#- the `config.get` merge option is not available for `ssh` or `unknown` salt command types #} +{%- else %} + {%- if _config['merge_strategy'] %} + {%- do salt['log.error']("map.jinja: the 'merge' option of 'config.get' is skipped when the salt command type is '" ~ cli ~ "'") %} + {%- endif %} + {%- do _config.update({'merge_opt': {}, + 'merge_msg': ''}) %} +{%- endif %} + + +{#- process each `map.jinja` source #} +{%- for map_source in map_sources %} + {%- if map_source in ['config_get', 'config_get_lookup'] %} + {%- set _config_key = {'config_get': tplroot, + 'config_get_lookup': tplroot ~ ':lookup'}.get(map_source) %} + {%- set _config_type = {'config_get': 'configuration', + 'config_get_lookup': 'lookup'}.get(map_source) %} + + {%- do salt['log.debug']("map.jinja: retrieve formula " ~ _config_type + ~ " with 'config.get'" + ~ _config['merge_msg'] + ) %} + {%- set _config_get = salt['config.get'](_config_key, default={}, **_config['merge_opt']) %} + + {#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #} + {%- set _strategy = _config['merge_strategy'] | default('smart', boolean=True) %} + {%- do salt['log.debug']("map.jinja: merge formula " ~ _config_type + ~ " retrieved with 'config.get'" + ~ ", merge: strategy='" ~ _strategy + ~ "', lists='" ~ _config['merge_lists'] ~ "'" + ) %} + {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'], + _config_get, + strategy=_strategy, + merge_lists=_config['merge_lists'])}) + %} + {%- else %} + {#- Lookup the grain/pillar/... #} + {#- Fallback to use the source name as a direct filename #} + {%- set map_values = salt['config.get'](map_source, []) %} + + {#- Mangle `map_source` to use it as literal path #} + {%- if map_values|length == 0 %} + {%- set map_source_parts = map_source.split('/') %} + {%- set map_source = map_source_parts[0:-1] | join('/') %} + {%- set map_values = map_source_parts[-1].rstrip('.yaml') %} + {%- endif %} + + {#- Some configuration return list #} + {%- if map_values is string %} + {%- set map_values = [map_values] %} + {%- endif %} + + {%- for map_value in map_values %} + {%- set yamlfile = map_sources_dir | path_join(map_source, map_value ~ '.yaml') %} + {%- do salt['log.debug']('map.jinja: load parameters from file ' ~ yamlfile) %} + {%- load_yaml as loaded_values %} + {%- include yamlfile ignore missing %} + {%- endload %} + + {%- if loaded_values %} + {#- Merge loaded values on the stack #} + {%- do salt['log.debug']('map.jinja: merge parameters from ' ~ yamlfile) %} + {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'], + loaded_values.get('values', {}), + strategy=loaded_values.get('strategy', 'smart'), + merge_lists=loaded_values.get('merge_lists', False) | to_bool)}) + %} + {%- endif %} + {%- endfor %} + {%- endif %} +{%- endfor %} {#- Change **TEMPLATE** to match with your formula's name and then remove this line #} -{%- set TEMPLATE = config %} +{%- do salt['log.debug']('map.jinja: save parameters in variable "TEMPLATE"') %} +{%- set TEMPLATE = _config['stack'] %} {#- Post-processing for specific non-YAML customisations #} {%- if grains.os == 'MacOS' %} From 9d27614caea8b9ec6970ae9d603accddb3efbd1a Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:39:39 +0100 Subject: [PATCH 3/9] feat(map.jinja): `defaults.yaml` must be under `parameters/` * TEMPLATE/parameters/defaults.yaml: remove the top level key `TEMPLATE` and add a `map.jinja:sources` configuration for tests. --- TEMPLATE/defaults.yaml | 15 -------------- TEMPLATE/parameters/defaults.yaml | 34 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) delete mode 100644 TEMPLATE/defaults.yaml create mode 100644 TEMPLATE/parameters/defaults.yaml diff --git a/TEMPLATE/defaults.yaml b/TEMPLATE/defaults.yaml deleted file mode 100644 index a5c495e3..00000000 --- a/TEMPLATE/defaults.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml ---- -TEMPLATE: - pkg: - name: TEMPLATE - rootgroup: root - config: '/etc/TEMPLATE' - service: - name: TEMPLATE - subcomponent: - config: '/etc/TEMPLATE-subcomponent-formula.conf' - # Just here for testing - added_in_defaults: defaults_value - winner: defaults diff --git a/TEMPLATE/parameters/defaults.yaml b/TEMPLATE/parameters/defaults.yaml new file mode 100644 index 00000000..4cb9642d --- /dev/null +++ b/TEMPLATE/parameters/defaults.yaml @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +values: + pkg: + name: TEMPLATE + rootgroup: root + config: '/etc/TEMPLATE' + service: + name: TEMPLATE + subcomponent: + config: '/etc/TEMPLATE-subcomponent-formula.conf' + + # Just here for testing + added_in_defaults: defaults_value + winner: defaults + # Configure map.jinja to load formula parameters values and merge them + # in the order from most generic (first) to most specific (last). + # Each source can define the merging `strategy` which default to the + # `smart` strategy of `salt.slsutil.merge`. + # Note: Any value not evaluated by `config.get` will be used literally. + # This can be used to set custom paths, as many levels deep as required. + map_jinja: + sources: + - osarch + - os_family + - os + - osfinger + - roles + - config_get_lookup + - config_get + - id + - any/path/can/be/used/here.yaml +... From fed0200bedd98c6f8a251586b710a116fbdad8ef Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:40:09 +0100 Subject: [PATCH 4/9] feat(map.jinja): split `osarchmap.yaml` under `parameters/osarch/` --- TEMPLATE/osarchmap.yaml | 35 ------------------------- TEMPLATE/parameters/osarch/386.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/amd64.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/arm64.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/armv6l.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/armv7l.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/ppc64le.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/s390x.yaml | 14 ++++++++++ TEMPLATE/parameters/osarch/x86_64.yaml | 14 ++++++++++ 9 files changed, 112 insertions(+), 35 deletions(-) delete mode 100644 TEMPLATE/osarchmap.yaml create mode 100644 TEMPLATE/parameters/osarch/386.yaml create mode 100644 TEMPLATE/parameters/osarch/amd64.yaml create mode 100644 TEMPLATE/parameters/osarch/arm64.yaml create mode 100644 TEMPLATE/parameters/osarch/armv6l.yaml create mode 100644 TEMPLATE/parameters/osarch/armv7l.yaml create mode 100644 TEMPLATE/parameters/osarch/ppc64le.yaml create mode 100644 TEMPLATE/parameters/osarch/s390x.yaml create mode 100644 TEMPLATE/parameters/osarch/x86_64.yaml diff --git a/TEMPLATE/osarchmap.yaml b/TEMPLATE/osarchmap.yaml deleted file mode 100644 index ab3bc1f4..00000000 --- a/TEMPLATE/osarchmap.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['osarch'] based logic. -# You just need to add the key:values for an `osarch` that differ -# from `defaults.yaml`. -# Only add an `osarch` which is/will be supported by the formula. -# -# If you do not need to provide defaults via the `osarch` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osarch: {} ---- -amd64: - arch: amd64 - -x86_64: - arch: amd64 - -386: - arch: 386 - -arm64: - arch: arm64 - -armv6l: - arch: armv6l - -armv7l: - arch: armv7l - -ppc64le: - arch: ppc64le - -s390x: - arch: s390x diff --git a/TEMPLATE/parameters/osarch/386.yaml b/TEMPLATE/parameters/osarch/386.yaml new file mode 100644 index 00000000..377787a7 --- /dev/null +++ b/TEMPLATE/parameters/osarch/386.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == 386. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: 386 +... diff --git a/TEMPLATE/parameters/osarch/amd64.yaml b/TEMPLATE/parameters/osarch/amd64.yaml new file mode 100644 index 00000000..5eee548e --- /dev/null +++ b/TEMPLATE/parameters/osarch/amd64.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == amd64. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: amd64 +... diff --git a/TEMPLATE/parameters/osarch/arm64.yaml b/TEMPLATE/parameters/osarch/arm64.yaml new file mode 100644 index 00000000..8846d066 --- /dev/null +++ b/TEMPLATE/parameters/osarch/arm64.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == arm64. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: arm64 +... diff --git a/TEMPLATE/parameters/osarch/armv6l.yaml b/TEMPLATE/parameters/osarch/armv6l.yaml new file mode 100644 index 00000000..c59440b4 --- /dev/null +++ b/TEMPLATE/parameters/osarch/armv6l.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == armv6l. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: armv6l +... diff --git a/TEMPLATE/parameters/osarch/armv7l.yaml b/TEMPLATE/parameters/osarch/armv7l.yaml new file mode 100644 index 00000000..b949603d --- /dev/null +++ b/TEMPLATE/parameters/osarch/armv7l.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == armv7l. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: armv7l +... diff --git a/TEMPLATE/parameters/osarch/ppc64le.yaml b/TEMPLATE/parameters/osarch/ppc64le.yaml new file mode 100644 index 00000000..eab2c457 --- /dev/null +++ b/TEMPLATE/parameters/osarch/ppc64le.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == ppc64le. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: ppc64le +... diff --git a/TEMPLATE/parameters/osarch/s390x.yaml b/TEMPLATE/parameters/osarch/s390x.yaml new file mode 100644 index 00000000..c8651c6d --- /dev/null +++ b/TEMPLATE/parameters/osarch/s390x.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == s390x. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: s390x +... diff --git a/TEMPLATE/parameters/osarch/x86_64.yaml b/TEMPLATE/parameters/osarch/x86_64.yaml new file mode 100644 index 00000000..2c009dd1 --- /dev/null +++ b/TEMPLATE/parameters/osarch/x86_64.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osarch') == x86_64. +# You just need to add the key:values for this `osarch` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osarch` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + arch: amd64 +... From 70dffa45a0cec13fef12a22e93f51c3c99f087e1 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:42:23 +0100 Subject: [PATCH 5/9] feat(map.jinja): split `osfamilymap.yaml` under `parameters/os_family/` --- TEMPLATE/osfamilymap.yaml | 47 ---------------------- TEMPLATE/parameters/os_family/Arch.yaml | 17 ++++++++ TEMPLATE/parameters/os_family/Debian.yaml | 16 ++++++++ TEMPLATE/parameters/os_family/FreeBSD.yaml | 14 +++++++ TEMPLATE/parameters/os_family/OpenBSD.yaml | 14 +++++++ TEMPLATE/parameters/os_family/RedHat.yaml | 16 ++++++++ TEMPLATE/parameters/os_family/Suse.yaml | 15 +++++++ 7 files changed, 92 insertions(+), 47 deletions(-) delete mode 100644 TEMPLATE/osfamilymap.yaml create mode 100644 TEMPLATE/parameters/os_family/Arch.yaml create mode 100644 TEMPLATE/parameters/os_family/Debian.yaml create mode 100644 TEMPLATE/parameters/os_family/FreeBSD.yaml create mode 100644 TEMPLATE/parameters/os_family/OpenBSD.yaml create mode 100644 TEMPLATE/parameters/os_family/RedHat.yaml create mode 100644 TEMPLATE/parameters/os_family/Suse.yaml diff --git a/TEMPLATE/osfamilymap.yaml b/TEMPLATE/osfamilymap.yaml deleted file mode 100644 index 6b473a88..00000000 --- a/TEMPLATE/osfamilymap.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['os_family'] based logic. -# You just need to add the key:values for an `os_family` that differ -# from `defaults.yaml` + `osarch.yaml`. -# Only add an `os_family` which is/will be supported by the formula. -# -# If you do not need to provide defaults via the `os_family` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osfamilymap: {} ---- -Debian: - pkg: - name: TEMPLATE-debian - config: /etc/TEMPLATE.d/custom.conf - -RedHat: - pkg: - name: TEMPLATE-redhat - config: /etc/TEMPLATE.conf - -Suse: - pkg: - name: TEMPLATE-suse - -Gentoo: {} - -Arch: - pkg: - name: TEMPLATE-arch - service: - name: service-arch - -Alpine: {} - -FreeBSD: - rootgroup: wheel - -OpenBSD: - rootgroup: wheel - -Solaris: {} - -Windows: {} - -MacOS: {} diff --git a/TEMPLATE/parameters/os_family/Arch.yaml b/TEMPLATE/parameters/os_family/Arch.yaml new file mode 100644 index 00000000..266641fe --- /dev/null +++ b/TEMPLATE/parameters/os_family/Arch.yaml @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Arch. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-arch + service: + name: service-arch +... diff --git a/TEMPLATE/parameters/os_family/Debian.yaml b/TEMPLATE/parameters/os_family/Debian.yaml new file mode 100644 index 00000000..c3a30541 --- /dev/null +++ b/TEMPLATE/parameters/os_family/Debian.yaml @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Debian. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-debian + config: /etc/TEMPLATE.d/custom.conf +... diff --git a/TEMPLATE/parameters/os_family/FreeBSD.yaml b/TEMPLATE/parameters/os_family/FreeBSD.yaml new file mode 100644 index 00000000..1514cd0d --- /dev/null +++ b/TEMPLATE/parameters/os_family/FreeBSD.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == FreeBSD. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + rootgroup: wheel +... diff --git a/TEMPLATE/parameters/os_family/OpenBSD.yaml b/TEMPLATE/parameters/os_family/OpenBSD.yaml new file mode 100644 index 00000000..4feba1d9 --- /dev/null +++ b/TEMPLATE/parameters/os_family/OpenBSD.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == OpenBSD. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + rootgroup: wheel +... diff --git a/TEMPLATE/parameters/os_family/RedHat.yaml b/TEMPLATE/parameters/os_family/RedHat.yaml new file mode 100644 index 00000000..9af605e9 --- /dev/null +++ b/TEMPLATE/parameters/os_family/RedHat.yaml @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == RedHat. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-redhat + config: /etc/TEMPLATE.conf +... diff --git a/TEMPLATE/parameters/os_family/Suse.yaml b/TEMPLATE/parameters/os_family/Suse.yaml new file mode 100644 index 00000000..07bb2416 --- /dev/null +++ b/TEMPLATE/parameters/os_family/Suse.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Suse. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-suse +... From fff90fb266b6fac4b19dbc3ecffd31738d3ca7f4 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:43:42 +0100 Subject: [PATCH 6/9] feat(map.jinja): split `osmap.yaml` under `parameters/os/` --- TEMPLATE/osmap.yaml | 40 ------------------------------ TEMPLATE/parameters/os/Fedora.yaml | 17 +++++++++++++ TEMPLATE/parameters/os/Ubuntu.yaml | 16 ++++++++++++ 3 files changed, 33 insertions(+), 40 deletions(-) delete mode 100644 TEMPLATE/osmap.yaml create mode 100644 TEMPLATE/parameters/os/Fedora.yaml create mode 100644 TEMPLATE/parameters/os/Ubuntu.yaml diff --git a/TEMPLATE/osmap.yaml b/TEMPLATE/osmap.yaml deleted file mode 100644 index 9e2d89e0..00000000 --- a/TEMPLATE/osmap.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['os'] based logic. -# You just need to add the key:values for an `os` that differ -# from `defaults.yaml` + `osarch.yaml` + `os_family.yaml`. -# Only add an `os` which is/will be supported by the formula. -# -# If you do not need to provide defaults via the `os` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osmap: {} ---- -# os_family: Debian -Ubuntu: - pkg: - name: TEMPLATE-ubuntu - config: /etc/TEMPLATE.d/custom-ubuntu.conf -Raspbian: {} - -# os_family: RedHat -Fedora: - pkg: - name: TEMPLATE-fedora - service: - name: service-fedora -CentOS: {} -Amazon: {} - -# os_family: Suse -SUSE: {} -openSUSE: {} - -# os_family: Gentoo -Funtoo: {} - -# os_family: Arch -Manjaro: {} - -# os_family: Solaris -SmartOS: {} diff --git a/TEMPLATE/parameters/os/Fedora.yaml b/TEMPLATE/parameters/os/Fedora.yaml new file mode 100644 index 00000000..ebbe2e46 --- /dev/null +++ b/TEMPLATE/parameters/os/Fedora.yaml @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os') == Fedora. +# You just need to add the key:values for this `os` that differ +# from `defaults.yaml` + `.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-fedora + service: + name: service-fedora +... diff --git a/TEMPLATE/parameters/os/Ubuntu.yaml b/TEMPLATE/parameters/os/Ubuntu.yaml new file mode 100644 index 00000000..324cb3d3 --- /dev/null +++ b/TEMPLATE/parameters/os/Ubuntu.yaml @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os') == Ubuntu. +# You just need to add the key:values for this `os` that differ +# from `defaults.yaml` + `.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-ubuntu + config: /etc/TEMPLATE.d/custom-ubuntu.conf +... From 128fe8752b54a8029e1a9e896c2f723e0d587744 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 18:45:05 +0100 Subject: [PATCH 7/9] feat(map.jinja): split `osfingermap.yaml` under `parameters/osfinger/` --- TEMPLATE/osfingermap.yaml | 49 ------------------- TEMPLATE/parameters/osfinger/CentOS-6.yaml | 16 ++++++ .../parameters/osfinger/Ubuntu-18.04.yaml | 14 ++++++ 3 files changed, 30 insertions(+), 49 deletions(-) delete mode 100644 TEMPLATE/osfingermap.yaml create mode 100644 TEMPLATE/parameters/osfinger/CentOS-6.yaml create mode 100644 TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml diff --git a/TEMPLATE/osfingermap.yaml b/TEMPLATE/osfingermap.yaml deleted file mode 100644 index 7bd9ba7a..00000000 --- a/TEMPLATE/osfingermap.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['osfinger'] based logic. -# You just need to add the key:values for an `osfinger` that differ -# from `defaults.yaml` + `osarch.yaml` + `os_family.yaml` + `osmap.yaml`. -# Only add an `osfinger` which is/will be supported by the formula. -# -# If you do not need to provide defaults via the `os_finger` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osfingermap: {} ---- -# os: Debian -Debian-10: {} -Debian-9: {} -Debian-8: {} - -# os: Ubuntu -Ubuntu-18.04: - config: /etc/TEMPLATE.d/custom-ubuntu-18.04.conf -Ubuntu-16.04: {} - -# os: Fedora -Fedora-31: {} -Fedora-30: {} - -# os: CentOS -CentOS Linux-8: {} -CentOS Linux-7: {} -CentOS-6: - pkg: - name: TEMPLATE-centos-6 - config: /etc/TEMPLATE.d/custom-centos-6.conf - -# os: Amazon -Amazon Linux-2: {} -Amazon Linux AMI-2018: {} - -# os: SUSE -Leap-15: {} - -# os: FreeBSD -FreeBSD-12: {} - -# os: Windows -Windows-8.1: {} - -# os: Gentoo -Gentoo-2: {} diff --git a/TEMPLATE/parameters/osfinger/CentOS-6.yaml b/TEMPLATE/parameters/osfinger/CentOS-6.yaml new file mode 100644 index 00000000..714e3ede --- /dev/null +++ b/TEMPLATE/parameters/osfinger/CentOS-6.yaml @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osfinger') == CentOS-6. +# You just need to add the key:values for this `osfinger` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osfinger` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + pkg: + name: TEMPLATE-centos-6 + config: /etc/TEMPLATE.d/custom-centos-6.conf +... diff --git a/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml b/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml new file mode 100644 index 00000000..67048dd2 --- /dev/null +++ b/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osfinger') == Ubuntu-18.04. +# You just need to add the key:values for this `osfinger` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osfinger` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + config: /etc/TEMPLATE.d/custom-ubuntu-18.04.conf +... From 4cd16cb20a602d5b7689438cd3a0a918010d2e6f Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 19:08:01 +0100 Subject: [PATCH 8/9] chore(formula): the use of `traverse` jinja filter requires 2018.3.3 --- FORMULA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FORMULA b/FORMULA index 0cdc69f1..d8916609 100644 --- a/FORMULA +++ b/FORMULA @@ -3,7 +3,7 @@ os: Debian, Ubuntu, Raspbian, RedHat, Fedora, CentOS, Amazon, Suse, openSUSE, Ge os_family: Debian, RedHat, Suse, Gentoo, Arch, Alpine, FreeBSD, OpenBSD, Solaris, Windows, MacOS version: 4.0.5 release: 1 -minimum_version: 2017.7 +minimum_version: 2018.3 summary: TEMPLATE formula description: Formula to use as a template for other formulas top_level_dir: TEMPLATE From 30961a781181d97207acc3948f9136acf45e75d6 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 19:08:05 +0100 Subject: [PATCH 9/9] docs(map.jinja): describe the use and configuration of `map.jinja` --- docs/README.rst | 6 +- docs/map.jinja.rst | 214 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 docs/map.jinja.rst diff --git a/docs/README.rst b/docs/README.rst index 7e252f8e..7d3f2725 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -33,7 +33,11 @@ which contains the currently released version. This formula is versioned accordi See `Formula Versioning Section `_ for more details. -If you need (non-default) configuration, please pay attention to the ``pillar.example`` file and/or `Special notes`_ section. +If you need (non-default) configuration, please refer to: + +- the ``pillar.example`` file +- `how to configure the formula with map.jinja `_ +- the `Special notes`_ section Contributing to this repo ------------------------- diff --git a/docs/map.jinja.rst b/docs/map.jinja.rst new file mode 100644 index 00000000..e2cf38d8 --- /dev/null +++ b/docs/map.jinja.rst @@ -0,0 +1,214 @@ +.. _map.jinja: + +``map.jinja``: gather formula configuration values +================================================== + +The `documentation`_ gives a quick view of the use of a ``map.jinja`` to gather parameters values for a formula. + +Current best practice is to let ``map.jinja`` handle parameters from all sources, to minimise the use of pillars, grains or configuration from ``sls`` files and templates directly. + + +.. contents:: **Table of Contents** + + +How to set configuration values of a formula +-------------------------------------------- + +The ``map.jinja`` file uses several sources where to lookup parameter values: + +- Multiple YAML files organised in the `fileserver`_ under the formula ``parameters/`` directory +- Configuration collected by `salt['config.get']`_ + +As `pillars`_ are rendered on the Salt master for every minion, this increases the load on the master as the pillar values and the number of minions grow. + +As a good practice, you should: + +- store non-secret data in YAML files distributed by the `fileserver`_ +- store secret data in: + + - `pillars`_ (and look for the use of something like `pillar.vault`_) + - `SDB`_ (and look for the use of something like `sdb.vault`_) + + +Format of configuration YAML files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When you write a new YAML file, note that it must conform to the following layout: + +- a mandatory ``values`` key to store the configuration values +- an optional ``strategy`` key to configure the merging strategy of ``salt.slsutil.merge``, for example ``strategy: 'recurse'``, the default is ``smart`` +- an optional ``merge_lists`` key to configure the merging of list for the ``recurse`` and ``overwrite``, for example ``merge_lists: 'true'`` + +Here is a valid example: + +.. code-block:: yaml + + --- + strategy: 'recurse' + merge_lists: 'false' + values: + pkg: + name: 'some-package' + config: '/path/to/a/configuration/file' + ... + + +Sources of configuration values +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +Configuring ``map.jinja`` sources +````````````````````````````````` + +At import time, ``map.jinja`` will configure itself the sources from where to import configuration values: + +#. builtin default + + #. ``osarch``: grain of the CPU architecture of the minion + #. ``os_family``: grain of the family of the operating system (e.g. ``Debian`` for an ``Ubuntu``) + #. ``os``: grain of the name of the operating system (e.g. ``Ubuntu``) + #. ``osfinger``: grain the concatenation of the operating system name and it's version string (e.g. ``Debian-10``) + #. ``config_get_lookup`` formula `lookup` table retrived by `salt['config.get']`_ + #. ``config_get``: formula global configuration retrived by `salt['config.get']`_ + #. ``id``: grain of the ``ID`` of the minion + +#. `defaults.yaml`: optionally define a formula specific `map_jinja:sources` for the author of the formula +#. global configuration lookup `map_jinja:sources` +#. formula specific `:map_jinja:sources` + +To configure ``map.jinja``, we need to set ``map_jinja:sources`` either globally or per formula. + + +Loading values from the configuration sources +````````````````````````````````````````````` + +For each configuration source defined in ``map_jinja:sources``, ``map.jinja`` will + +#. load the values from: + + - ``salt['config.get'](tplroot ~ ':lookup')`` if the key is ``config_get_lookup`` + - ``salt['config.get'](tplroot)`` if the key is ``config_get`` + - ``parameters//.yaml`` for the configuration named ```` with value ```` + - ``parameters/.yaml`` if the configuration lookup fails, the key name is used as a litteral string path of a custom path to a YAML file, for example: ``any/path/can/be/used/here.yaml`` will result in the loading of ``parameters/any/path/can/be/used/here.yaml`` + +#. merge the loaded values with the previous ones using `salt.slsutil.merge`_ + +There will be no error if a YAML file does not exists, they are all optional (except ``defaults.yaml``). + + +Configuration values from ``salt['config.get']`` +```````````````````````````````````````````````` + +If at least one of ``config_get_lookup`` or ``config_get`` source is declared in ``map_jinja:sources``, then the corresponding configuration is looked up with `salt['config.get']`_ and merged with the previously aggregated values. + +You can configure the ``merge`` option of `salt['config.get']`_ by defining per formula ``strategy`` configuration key with one of the following values: + +- ``recurse`` merge recursively dictionaries. Non dictionary values replace already defined values +- ``overwrite`` new value completely replace old ones + +By default, no merging is done, the first value found is returned. + + +Global view of the order of preferences +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To make resumé, here is a complete example of the load order of formula configuration values for an ``AMD64`` ``Ubuntu 18.04`` minion named ``minion1.example.net`` for the ``mysql`` formula: + +#. ``parameters/defaults.yaml`` +#. ``parameters/osarch/amd64.yaml`` +#. ``parameters/os_family/Debian.yaml`` +#. ``parameters/os/Ubunta.yaml`` +#. ``parameters/osfinger/Ubunta-18.04.yaml`` +#. ``salt['config.get']('mysql:lookup')`` +#. ``salt['config.get']('mysql')`` +#. ``parameters/id/minion1.example.net`` + +Remember that the order is important, for example, the value of ``key1:subkey1`` loaded from ``parameters/os_family/Debian.yaml`` is overridden by a value loaded from ``parameters/id/minion1.example.net``. + + + +Use formula configuration values in `sls` +----------------------------------------- + +The good practice for ``map.jinja`` are: + +- to be located at the root of the formula named directory (e.g. ``mysql-formula/mysql/map.jinja``) +- to export a variable with the same name than the formula itself. As an example, for ``mysql-formula`` it will be ``mysql``. + +Here is the best way to use it in an ``sls`` file: + +.. code-block:: sls + + {#- Get the `tplroot` from `tpldir` #} + {%- set tplroot = tpldir.split('/')[0] %} + {%- from tplroot | path_join('map.jinja') import TEMPLATE with context %} + + test-does-nothing-but-display-TEMPLATE-as-json: + test.nop: + - name: {{ TEMPLATE | json }} + + + +Use formula configuration values in templates +--------------------------------------------- + +When you need to process salt templates, you should avoid calling ``salt['config.get']`` (or ``salt['pillar.get']`` and ``salt['grains.get']``) directly from the template. All the needed values should be available within the variable exported by ``map.jinja``. + +Here is an example based on ``template-formula/TEMPLATE/config/file.sls`` + +.. code-block:: sls + + # -*- coding: utf-8 -*- + # vim: ft=sls + + {#- Get the `tplroot` from `tpldir` #} + {%- set tplroot = tpldir.split('/')[0] %} + {%- set sls_package_install = tplroot ~ '.package.install' %} + {%- from tplroot ~ "/map.jinja" import TEMPLATE with context %} + {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} + + include: + - {{ sls_package_install }} + + TEMPLATE-config-file-file-managed: + file.managed: + - name: {{ TEMPLATE.config }} + - source: {{ files_switch(['example.tmpl'], + lookup='TEMPLATE-config-file-file-managed' + ) + }} + - mode: 644 + - user: root + - group: {{ TEMPLATE.rootgroup }} + - makedirs: True + - template: jinja + - require: + - sls: {{ sls_package_install }} + - context: + TEMPLATE: {{ TEMPLATE | json }} + +This ``sls`` file expose a ``TEMPLATE`` context variable to the jinja template which could be used like this: + +.. code-block:: jinja + + ######################################################################## + # File managed by Salt at <{{ source }}>. + # Your changes will be overwritten. + ######################################################################## + + This is another example file from SaltStack template-formula. + + # This is here for testing purposes + {{ TEMPLATE | json }} + + winner of the merge: {{ TEMPLATE['winner'] }} + + +.. _documentation: https://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html#writing-formulas +.. _fileserver: https://docs.saltstack.com/en/latest/ref/configuration/master.html#std:conf_master-fileserver_backend +.. _salt['config.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.config.html#salt.modules.config.get +.. _pillar.vault: https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.vault.html +.. _pillars: https://docs.saltstack.com/en/latest/topics/pillar/ +.. _SDB: https://docs.saltstack.com/en/latest/topics/sdb/index.html +.. _sdb.vault: https://docs.saltstack.com/en/latest/ref/sdb/all/salt.sdb.vault.html +.. _salt.slsutil.merge: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.slsutil.html