feat(map): use targeting like syntax for configuration

The `config_get_lookup` and `config_get` sources lack flexibility.

It's not easy to query several pillars and/or grains keys with the
actual system. And the query method is forced to `config.get` without
being configurable by the user.

We define a mechanism to select `map.jinja` sources with similar
notation as the salt targeting system.

The `map.jinja` file uses several sources where to lookup parameter
values. The list of sources can be modified by two files:

1. a global salt://parameters/map_jinja.yaml
2. a per formula salt://{{ tplroot }}/parameters/map_jinja.yaml.

Each source definition has the form `<TYPE>:<OPTION>@<KEY>` where
`<TYPE>` can be one of:

- `Y` to load values from YAML files, this is the default when no type
  is defined
- `C` to lookup values with `config.get`
- `G` to lookup values with `grains.get`
- `I` to lookup values with `pillar.get`

The YAML type option can define the query method to lookup the key
value to build the file name:

- `C` to query with `config.get`, this is the default when to query
  method is defined
- `G` to query with `grains.get`
- `I` to query with `pillar.get`

The `C`, `G` or `I` types can define the `SUB` option to store values
in the sub key `mapdata.<key>` instead of directly in `mapdata`.

Finally, the `<KEY>` describe what to lookup to either build the YAML
filename or gather values using one of the query method.

BREAKING CHANGE: the configuration `map_jinja:sources` is only
                 configurable with `salt://parameters/map_jinja.yaml`
		 and `salt://{{ tplroot }}/parameters/map_jinja.yaml`

BREAKING CHANGE: the `map_jinja:config_get_roots` is replaced by
                 compound like `map_jinja:sources`

BREAKING CHANGE: the two `config_get_lookup` and `config_get` are
                 replaced by `C@<tplroot>:lookup` and `C@<tplroot>`
		 sources
This commit is contained in:
Daniel Dehennin 2021-01-11 17:31:22 +01:00
parent b9b8eb7618
commit 1be0d8725a
17 changed files with 382 additions and 164 deletions

View File

@ -8,69 +8,102 @@
{#- Where to lookup parameters source files #} {#- Where to lookup parameters source files #}
{%- set map_sources_dir = tplroot ~ "/parameters" %} {%- set map_sources_dir = tplroot ~ "/parameters" %}
{#- Load defaults first to allow per formula default map.jinja configuration #}
{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
{%- do salt["log.debug"](
"map.jinja: initialise parameters from "
~ _defaults_filename
) %}
{%- import_yaml _defaults_filename as default_settings %}
{#- List of sources to lookup for parameters #} {#- List of sources to lookup for parameters #}
{%- do salt["log.debug"]("map.jinja: lookup 'map_jinja' configuration sources") %}
{#- Fallback to previously used grains plus minion `id` #} {#- Fallback to previously used grains plus minion `id` #}
{%- set map_sources = [ {%- set map_sources = [
"osarch", "Y:G@osarch",
"os_family", "Y:G@os_family",
"os", "Y:G@os",
"osfinger", "Y:G@osfinger",
"config_get_lookup", "C@" ~ tplroot ~ ":lookup",
"config_get", "C@" ~ tplroot,
"id", "Y:G@id",
] %} ] %}
{#- Configure map.jinja from defaults.yaml #} {%- do salt["log.debug"](
{%- set map_sources = default_settings | traverse( "map.jinja: built-in configuration sources:\n"
"values:map_jinja:sources", ~ {"values": {
map_sources, "map_jinja": {"sources": map_sources}
}
}
| yaml(False)
) %} ) %}
{#- Lookup global sources #} {#- Allow centralised map.jinja configuration #}
{%- set map_sources = salt["config.get"]("map_jinja:sources", map_sources) %} {%- set _global_map_filename = "parameters/map_jinja.yaml" %}
{#- Lookup per formula sources #} {%- do salt["log.debug"](
{%- set map_sources = salt["config.get"]( "map.jinja: load global map.jinja values from "
tplroot ~ ":map_jinja:sources", ~ _global_map_filename
map_sources,
) %} ) %}
{%- load_yaml as global_map_settings %}
{%- include _global_map_filename ignore missing %}
{%- endload %}
{%- if global_map_settings %}
{%- do salt["log.debug"](
"map.jinja: configure sources from global map.jinja configuration "
~ _global_map_filename
~ ":\n"
~ {"map_jinja": global_map_settings}
| yaml(False)
) %}
{%- set map_sources = global_map_settings
| traverse(
"values:sources",
map_sources,
) %}
{%- endif %}
{#- Allow per formula map.jinja configuration #}
{%- set _map_filename = map_sources_dir ~ "/map_jinja.yaml" %}
{%- do salt["log.debug"](
"map.jinja: load per formula map.jinja values from "
~ _map_filename
) %}
{%- load_yaml as map_settings %}
{%- include _map_filename ignore missing %}
{%- endload %}
{%- if map_settings %}
{%- do salt["log.debug"](
"map.jinja: configure sources from formula map.jinja configuration "
~ _map_filename
~ ":\n"
~ {"map_jinja": map_settings}
| yaml(False)
) %}
{%- set map_sources = map_settings
| traverse(
"values:sources",
map_sources,
) %}
{%- endif %}
{%- do salt["log.debug"]( {%- do salt["log.debug"](
"map.jinja: load parameters with sources from " "map.jinja: load parameters from sources:\n"
~ map_sources ~ map_sources
| yaml(False)
) %} ) %}
{#- Lookup with `config.get` from configurable roots #} {#- Load formula defaults values #}
{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
{%- do salt["log.debug"]( {%- do salt["log.debug"](
"map.jinja: initialise 'config.get' roots with 'tplroot' " "map.jinja: load per formula default values from "
~ tplroot ~ _defaults_filename
) %} ) %}
{%- set config_get_roots = [tplroot] %} {%- load_yaml as default_settings %}
{#- Configure `config.get` from defaults.yaml #} {%- include _defaults_filename ignore missing %}
{%- set config_get_roots = default_settings | traverse( {%- endload %}
"values:map_jinja:config_get_roots",
config_get_roots {%- if not default_settings %}
) %} {%- set default_settings = {'values': {} } %}
{#- Lookup global `config.get` roots #} {%- endif %}
{%- set config_get_roots = salt["config.get"](
"map_jinja:config_get_roots", {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
config_get_roots {%- do default_settings["values"].update(
) %} {
{#- Lookup per formula `config.get` roots #} "map_jinja": map_settings
{%- set config_get_roots = salt["config.get"]( | traverse("values", {})
tplroot ~ ":map_jinja:config_get_roots", }
config_get_roots,
) %}
{%- do salt["log.debug"](
"map.jinja: load parameters with 'config.get' from roots "
~ config_get_roots
) %} ) %}
{#- Work around assignment inside for loop #} {#- Work around assignment inside for loop #}
@ -106,63 +139,115 @@
) %} ) %}
{%- endif %} {%- endif %}
{%- set query_map = {
"C": "config.get",
"G": "grains.get",
"I": "pillar.get",
} %}
{#- process each `map.jinja` source #} {#- Process each `map.jinja` source #}
{#- each source has a type: #}
{#- - `Y` to load values from YAML files (the default when no type is set) #}
{#- - `C` to lookup values with `config.get` #}
{#- - `G` to lookup values with `grains.get` #}
{#- - `I` to lookup values with `pillar.get` #}
{#- The YAML type option can define query type to build the file name: #}
{#- - `C` for query with `config.get` (the default when to query type is set) #}
{#- - `G` for query with `grains.get` #}
{#- - `I` for query with `pillar.get` #}
{#- The `C`, `G` or `I` types can define the `SUB` option #}
{#- to merge values in the sub key `mapdata.<key>` instead of directly in `mapdata` #}
{%- for map_source in map_sources %} {%- for map_source in map_sources %}
{%- if map_source in ["config_get", "config_get_lookup"] %} {%- set source_parts = map_source.split('@') %}
{%- for _config_root in config_get_roots %} {%- if source_parts|length == 1 %}
{%- set _config_key = { {#- By default we load YAML files for config looked up by `config.get` #}
"config_get": _config_root, {%- set source_type = "Y" %}
"config_get_lookup": _config_root ~ ":lookup", {%- set query_type = "C" %}
}.get(map_source) %} {%- set source_key = map_source %}
{%- elif source_parts[0][0] == "Y" %}
{%- set source_type = "Y" %}
{%- set query_type = source_parts[0].split(':')[1] | default("C") %}
{%- set source_key = source_parts[1] %}
{%- elif source_parts[0][0] in query_map.keys() %}
{%- set source_type = source_parts[0].split(':') | first %}
{%- set query_type = source_type %}
{%- set is_sub_key = source_parts[0].split(':')[1] | default(False) == "SUB" %}
{%- set source_key = source_parts[1] %}
{%- endif %}
{%- do salt["log.debug"]( {%- set query_method = query_map[query_type] %}
"map.jinja: retrieve '"
~ _config_key
~ "' with 'config.get'"
~ _config["merge_msg"]
) %}
{%- set _config_get = salt["config.get"](
_config_key, default={}, **_config["merge_opt"]
) %}
{#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #} {%- if source_type in query_map.keys() %}
{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %} {#- Lookup source `<QUERY_METHOD>@key:to:query` #}
{%- do salt["log.debug"]( {%- if source_type == "C" %}
"map.jinja: merge '" {%- set merge_opts = _config["merge_opt"] %}
~ _config_key {%- set merge_msg = _config["merge_msg"] %}
~ "' retrieved with 'config.get'" {%- else %}
~ ", merge: strategy='" {#- No merging strategy supported for `grains.get` and `pillar.get` #}
~ _strategy {%- set merge_opts = {} %}
~ "', lists='" {%- set merge_msg = "" %}
~ _config["merge_lists"] {%- endif %}
~ "'"
) %}
{#- Keep values under each root key when there are more than one #} {%- do salt["log.debug"](
{%- if config_get_roots|length > 1 %} "map.jinja: retrieve '"
{%- set _config_get = { _config_root: _config_get } %} ~ source_key
{%- endif %} ~ "' with '"
{%- do _config.update( ~ query_method
{ ~ "'"
"stack": salt["slsutil.merge"]( ~ merge_msg
_config["stack"], ) %}
_config_get, {%- set _config_get = salt[query_method](
strategy=_strategy, source_key,
merge_lists=_config["merge_lists"], default={},
) **merge_opts
} ) %}
) %}
{%- endfor %} {#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
{%- do salt["log.debug"](
"map.jinja: merge "
~ "sub key " * is_sub_key
~ "'"
~ source_key
~ "' retrieved with '"
~ query_method
~ "', merge: strategy='"
~ _strategy
~ "', lists='"
~ _config["merge_lists"]
~ "'"
) %}
{%- if is_sub_key %}
{#- Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
{%- set _config_get = { source_key.rstrip(':lookup'): _config_get } %}
{%- endif %}
{%- do _config.update(
{
"stack": salt["slsutil.merge"](
_config["stack"],
_config_get,
strategy=_strategy,
merge_lists=_config["merge_lists"],
)
}
) %}
{%- else %} {%- else %}
{#- Lookup the grain/pillar/... #} {#- Load YAML file matching the grain/pillar/... #}
{#- Fallback to use the source name as a direct filename #} {#- Fallback to use the source name as a direct filename #}
{%- set map_values = salt["config.get"](map_source, []) %} {%- do salt["log.debug"](
"map.jinja: lookup '"
~ source_key
~ "' with '"
~ query_method
~ "'"
) %}
{%- set map_values = salt[query_method](source_key, []) %}
{#- Mangle `map_source` to use it as literal path #} {#- Mangle `source_key` to use it as literal path #}
{%- if map_values | length == 0 %} {%- if map_values | length == 0 %}
{%- set map_source_parts = map_source.split("/") %} {%- set map_source_parts = source_key.split("/") %}
{%- set map_source = map_source_parts[0:-1] | join("/") %} {%- set source_key = map_source_parts[0:-1] | join("/") %}
{%- set map_values = map_source_parts[-1].rstrip(".yaml") %} {%- set map_values = map_source_parts[-1].rstrip(".yaml") %}
{%- endif %} {%- endif %}
@ -171,14 +256,20 @@
{%- set map_values = [map_values] %} {%- set map_values = [map_values] %}
{%- endif %} {%- endif %}
{#- `source_key` can be an empty string with literal path like `myconf.yaml` #}
{%- set yaml_dir = [
map_sources_dir,
source_key
]
| select
| join("/") %}
{%- for map_value in map_values %} {%- for map_value in map_values %}
{%- set yamlfile = [ {%- set yamlfile = [
map_sources_dir, yaml_dir,
map_source, map_value ~ ".yaml"
map_value ~ ".yaml",
] ]
| join("/") | join("/") %}
%}
{%- do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %} {%- do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
{%- load_yaml as loaded_values %} {%- load_yaml as loaded_values %}
{%- include yamlfile ignore missing %} {%- include yamlfile ignore missing %}

View File

@ -2,12 +2,6 @@
# vim: ft=yaml # vim: ft=yaml
--- ---
values: values:
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh: openssh:
sshd_enable: true sshd_enable: true
sshd_binary: /usr/sbin/sshd sshd_binary: /usr/sbin/sshd

View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
---
values:
sources:
- "Y:G@osarch"
- "Y:G@os_family"
- "Y:G@os"
- "Y:G@osfinger"
# Merge values from `config.get` under `mapdata.<key>` to keep
# compatibility with user pillars.
# The `<key>` and `<key>:lookup` are merged together
- "C:SUB@openssh:lookup"
- "C:SUB@openssh"
- "C:SUB@sshd_config:lookup"
- "C:SUB@sshd_config"
- "C:SUB@ssh_config:lookup"
- "C:SUB@ssh_config"
- "Y:G@id"

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false

View File

@ -3,10 +3,18 @@
--- ---
values: values:
map_jinja: map_jinja:
config_get_roots: sources:
- openssh - Y:G@osarch
- sshd_config - Y:G@os_family
- ssh_config - Y:G@os
- Y:G@osfinger
- C:SUB@openssh:lookup
- C:SUB@openssh
- C:SUB@sshd_config:lookup
- C:SUB@sshd_config
- C:SUB@ssh_config:lookup
- C:SUB@ssh_config
- Y:G@id
openssh: openssh:
absent_dsa_keys: false absent_dsa_keys: false
absent_ecdsa_keys: false absent_ecdsa_keys: false