refactor(map): load formula configuration with libmatchers
This commit is contained in:
Daniel Dehennin
parent
174bb68432
commit
ff6b56c4a4
@ -3,39 +3,104 @@
|
||||
|
||||
{#- Get the `tplroot` from `tpldir` #}
|
||||
{%- set tplroot = tpldir.split("/")[0] %}
|
||||
{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
|
||||
|
||||
{%- set _default_config_dirs = [
|
||||
"parameters/",
|
||||
tplroot ~ "/parameters"
|
||||
] %}
|
||||
|
||||
{%- macro mapstack(
|
||||
files,
|
||||
matchers,
|
||||
defaults=None,
|
||||
dirs=_default_config_dirs,
|
||||
log_prefix="libmapstack: "
|
||||
) %}
|
||||
{#-
|
||||
Load YAML files in the order of `files` and merge successively the
|
||||
values with `defaults` if the file exists.
|
||||
Load configuration in the order of `matchers` and merge
|
||||
successively the values with `defaults`.
|
||||
|
||||
The `matchers` are processed using `libmatchers.jinja` to select
|
||||
the configuration sources from where the values are loaded.
|
||||
|
||||
Parameters:
|
||||
- `files`: list of files to load
|
||||
|
||||
- `defaults`: default values to start the merging, they are
|
||||
considered built-ins
|
||||
- `matchers`: list of matchers in the form
|
||||
`[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`
|
||||
|
||||
- `defaults`: dictionary of default values to start the merging,
|
||||
they are considered built-ins. It must conform to the same
|
||||
layout as the YAML files: a mandatory `values` key and two
|
||||
optional `strategy` and `merge_lists` keys.
|
||||
|
||||
- `dirs`: list of directory where to look-up the configuration
|
||||
file matching the matchers, by default a global `salt://parameters/`
|
||||
and a per formula `salt://<tplroot>/parameters`
|
||||
|
||||
- `log_prefix`: prefix used in the log outputs, by default it is
|
||||
`libmapstack: `
|
||||
|
||||
Each YAML file must conform to the following layout:
|
||||
Example: On a Debian system with `roles=["nginx/server", "telegraf"]`
|
||||
|
||||
{%- set settings = mapstack(
|
||||
matchers=[
|
||||
"Y:G@os_family",
|
||||
"I@" ~ tplroot,
|
||||
"Y:C@roles",
|
||||
],
|
||||
dirs=["defaults", tplroot ~ "/parameters"],
|
||||
)
|
||||
| load_yaml %}
|
||||
|
||||
This will merge the values:
|
||||
|
||||
- starting with the default empty dictionary `{}` (no
|
||||
`defaults` parameter)
|
||||
|
||||
- from the YAML files
|
||||
|
||||
- `salt://defaults/os_family/Debian.yaml`
|
||||
|
||||
- `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`
|
||||
|
||||
- from the pillar `salt["pillar.get"](tplroot)`
|
||||
|
||||
- from the `nginx/server` YAML files:
|
||||
|
||||
- `salt://defaults/roles/nginx/server.yaml`
|
||||
|
||||
- `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`
|
||||
|
||||
- from the `telegraf` YAML files:
|
||||
|
||||
- `salt://defaults/roles/telegraf.yaml`
|
||||
|
||||
- `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`
|
||||
|
||||
Each YAML file and the `defaults` parameters must conform to the
|
||||
following layout:
|
||||
|
||||
- a mandatory `values` key to store the configuration values
|
||||
|
||||
- two optional keys to configure the use of `salt.slsutil.merge`
|
||||
|
||||
- an optional `strategy` key to configure the merging strategy,
|
||||
for example `strategy: 'recurse'`, the default is `smart`
|
||||
- an optional `strategy` key to configure the merging
|
||||
strategy, for example `strategy: 'recurse'`, the default is
|
||||
`smart`
|
||||
|
||||
- an optional `merge_lists` key to configure if lists should be
|
||||
merged or overridden for the `recurse` and `overwrite`
|
||||
- an optional `merge_lists` key to configure if lists should
|
||||
be merged or overridden for the `recurse` and `overwrite`
|
||||
strategies, for example `merge_lists: 'true'`
|
||||
#}
|
||||
{%- set stack = defaults | default({"values": {} }, boolean=True) %}
|
||||
|
||||
{#- Build configuration file names based on matchers #}
|
||||
{%- set matchers = parse_matchers(
|
||||
matchers,
|
||||
log_prefix=log_prefix
|
||||
)
|
||||
| load_yaml %}
|
||||
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "built-in configuration:\n"
|
||||
@ -43,74 +108,182 @@
|
||||
| yaml(False)
|
||||
) %}
|
||||
|
||||
{%- for yaml_filename in files %}
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "load configuration values from "
|
||||
~ yaml_filename
|
||||
) %}
|
||||
|
||||
{%- load_yaml as yaml_values %}
|
||||
{%- include yaml_filename ignore missing %}
|
||||
{%- endload %}
|
||||
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "loaded configuration values from "
|
||||
~ yaml_filename
|
||||
~ ":\n"
|
||||
~ yaml_values
|
||||
| yaml(False)
|
||||
) %}
|
||||
|
||||
{%- if yaml_values %}
|
||||
{#- Per YAML file `salt["slsutil.merge"]` options or use defaults #}
|
||||
{%- set _strategy = yaml_values
|
||||
{%- for param_dir in dirs %}
|
||||
{%- for matcher in matchers %}
|
||||
{#- `slsutil.merge` options from #}
|
||||
{#- 1. the `value` #}
|
||||
{#- 2. the `defaults` #}
|
||||
{#- 3. the built-in #}
|
||||
{%- set _strategy = matcher.value
|
||||
| traverse(
|
||||
"strategy",
|
||||
defaults
|
||||
| traverse(
|
||||
"strategy",
|
||||
defaults
|
||||
| traverse(
|
||||
"strategy",
|
||||
"smart"
|
||||
)
|
||||
) %}
|
||||
{%- set _merge_lists = yaml_values
|
||||
"smart"
|
||||
)
|
||||
) %}
|
||||
{%- set _merge_lists = matcher.value
|
||||
| traverse(
|
||||
"merge_lists",
|
||||
defaults
|
||||
| traverse(
|
||||
"merge_lists",
|
||||
defaults
|
||||
| traverse(
|
||||
"merge_lists",
|
||||
False
|
||||
)
|
||||
False
|
||||
)
|
||||
| to_bool %}
|
||||
)
|
||||
| to_bool %}
|
||||
|
||||
{#- Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
|
||||
{%- do stack.update(
|
||||
{
|
||||
"values": salt["slsutil.merge"](
|
||||
stack["values"],
|
||||
yaml_values
|
||||
| traverse("values", {}),
|
||||
strategy=_strategy,
|
||||
merge_lists=_merge_lists,
|
||||
)
|
||||
}
|
||||
) %}
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "merged configuration values from "
|
||||
~ yaml_filename
|
||||
~ ", merge: strategy='"
|
||||
~ _strategy
|
||||
~ "', merge_lists='"
|
||||
~ _merge_lists
|
||||
~ "':\n"
|
||||
~ {"values": stack["values"]}
|
||||
| yaml(False)
|
||||
) %}
|
||||
{%- endif %}
|
||||
{%- if matcher.type in query_map.keys() %}
|
||||
{#- No value is an empty list, must be a dict for `stack.update` #}
|
||||
{%- set normalized_value = matcher.value | default({}, boolean=True) %}
|
||||
|
||||
{#- Merge in `mapdata.<query>` instead of directly in `mapdata` #}
|
||||
{%- set is_sub_key = matcher.option | default(False) == "SUB" %}
|
||||
{%- if is_sub_key %}
|
||||
{#- Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
|
||||
{%- set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
|
||||
{%- else %}
|
||||
{%- set value = normalized_value %}
|
||||
{%- endif %}
|
||||
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "merge "
|
||||
~ "sub key " * is_sub_key
|
||||
~ "'"
|
||||
~ matcher.query
|
||||
~ "' retrieved with '"
|
||||
~ matcher.query_method
|
||||
~ "', merge: strategy='"
|
||||
~ _strategy
|
||||
~ "', lists='"
|
||||
~ _merge_lists
|
||||
~ "':\n"
|
||||
~ value
|
||||
| yaml(False)
|
||||
) %}
|
||||
|
||||
{%- do stack.update(
|
||||
{
|
||||
"values": salt["slsutil.merge"](
|
||||
stack["values"],
|
||||
value,
|
||||
strategy=_strategy,
|
||||
merge_lists=_merge_lists,
|
||||
)
|
||||
}
|
||||
) %}
|
||||
|
||||
{%- else %}
|
||||
{#- Load YAML file matching the grain/pillar/... #}
|
||||
{#- Fallback to use the source name as a direct filename #}
|
||||
|
||||
{%- if matcher.value | length == 0 %}
|
||||
{#- Mangle `matcher.value` to use it as literal path #}
|
||||
{%- set query_parts = matcher.query.split("/") %}
|
||||
{%- set yaml_dirname = query_parts[0:-1] | join("/") %}
|
||||
{%- set yaml_names = query_parts[-1] %}
|
||||
{%- else %}
|
||||
{%- set yaml_dirname = matcher.query %}
|
||||
{%- set yaml_names = matcher.value %}
|
||||
{%- endif %}
|
||||
|
||||
{#- Some configuration return list #}
|
||||
{%- if yaml_names is string %}
|
||||
{%- set yaml_names = [yaml_names] %}
|
||||
{%- endif %}
|
||||
|
||||
{#- `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
|
||||
{%- set yaml_dir = [
|
||||
param_dir,
|
||||
yaml_dirname
|
||||
]
|
||||
| select
|
||||
| join("/") %}
|
||||
|
||||
{%- for yaml_name in yaml_names %}
|
||||
{#- Make sure to have a `.yaml` extension #}
|
||||
{#- Use `.rpartition` to strip last `.yaml` in `dir.yaml/file.yaml` #}
|
||||
{%- set yaml_filename = [
|
||||
yaml_dir.rstrip("/"),
|
||||
yaml_name.rpartition(".yaml")
|
||||
| reject("equalto", ".yaml")
|
||||
| join
|
||||
~ ".yaml"
|
||||
]
|
||||
| select
|
||||
| join("/") %}
|
||||
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "load configuration values from "
|
||||
~ yaml_filename
|
||||
) %}
|
||||
{%- load_yaml as yaml_values %}
|
||||
{%- include yaml_filename ignore missing %}
|
||||
{%- endload %}
|
||||
|
||||
{%- if yaml_values %}
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "loaded configuration values from "
|
||||
~ yaml_name
|
||||
~ ":\n"
|
||||
~ yaml_values
|
||||
| yaml(False)
|
||||
) %}
|
||||
|
||||
{#- `slsutil.merge` options from #}
|
||||
{#- 1. the `value` #}
|
||||
{#- 2. the `defaults` #}
|
||||
{#- 3. the built-in #}
|
||||
{%- set _strategy = yaml_values
|
||||
| traverse(
|
||||
"strategy",
|
||||
defaults
|
||||
| traverse(
|
||||
"strategy",
|
||||
"smart"
|
||||
)
|
||||
) %}
|
||||
{%- set _merge_lists = yaml_values
|
||||
| traverse(
|
||||
"merge_lists",
|
||||
defaults
|
||||
| traverse(
|
||||
"merge_lists",
|
||||
False
|
||||
)
|
||||
)
|
||||
| to_bool %}
|
||||
{%- do stack.update(
|
||||
{
|
||||
"values": salt["slsutil.merge"](
|
||||
stack["values"],
|
||||
yaml_values
|
||||
| traverse("values", {}),
|
||||
strategy=_strategy,
|
||||
merge_lists=_merge_lists,
|
||||
)
|
||||
}
|
||||
) %}
|
||||
{%- do salt["log.debug"](
|
||||
log_prefix
|
||||
~ "merged configuration values from "
|
||||
~ yaml_name
|
||||
~ ", merge: strategy='"
|
||||
~ _strategy
|
||||
~ "', merge_lists='"
|
||||
~ _merge_lists
|
||||
~ "':\n"
|
||||
~ {"values": stack["values"]}
|
||||
| yaml(False)
|
||||
) %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- do salt["log.debug"](
|
||||
@ -121,7 +294,7 @@
|
||||
) %}
|
||||
|
||||
{#- Output stack as YAML, caller should use with something like #}
|
||||
{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
|
||||
{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
|
||||
{{ stack | yaml }}
|
||||
|
||||
{%- endmacro %}
|
||||
|
||||
@ -3,11 +3,10 @@
|
||||
|
||||
{#- Get the `tplroot` from `tpldir` #}
|
||||
{%- set tplroot = tpldir.split("/")[0] %}
|
||||
{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
|
||||
{%- from tplroot ~ "/libmapstack.jinja" import mapstack %}
|
||||
|
||||
{#- Where to lookup parameters source files #}
|
||||
{%- set map_sources_dir = tplroot ~ "/parameters" %}
|
||||
{%- set formula_param_dir = tplroot ~ "/parameters" %}
|
||||
|
||||
{#- List of sources to lookup for parameters #}
|
||||
{#- Fallback to previously used grains plus minion `id` #}
|
||||
@ -21,13 +20,8 @@
|
||||
"Y:G@id",
|
||||
] %}
|
||||
|
||||
{#- Allow centralised map.jinja configuration #}
|
||||
{%- set _global_param_filename = "parameters/map_jinja.yaml" %}
|
||||
{#- Allow per formula map.jinja configuration #}
|
||||
{%- set _formula_param_filename = map_sources_dir ~ "/map_jinja.yaml" %}
|
||||
|
||||
{%- set _map_settings = mapstack(
|
||||
files=[_global_param_filename, _formula_param_filename],
|
||||
matchers=["map_jinja.yaml"],
|
||||
defaults={
|
||||
"values": {"sources": map_sources}
|
||||
},
|
||||
@ -42,136 +36,26 @@
|
||||
| yaml(False)
|
||||
) %}
|
||||
|
||||
{#- Load formula defaults values #}
|
||||
{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
|
||||
{%- set default_settings = mapstack(
|
||||
files=[_defaults_filename],
|
||||
defaults={"values": {} },
|
||||
log_prefix="map.jinja defaults: ",
|
||||
{#- Load formula parameters values #}
|
||||
{%- set _formula_matchers = ["defaults.yaml"] + map_sources %}
|
||||
{%- set _formula_settings = mapstack(
|
||||
matchers=_formula_matchers,
|
||||
dirs=[formula_param_dir],
|
||||
defaults={
|
||||
"values": {},
|
||||
"merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
|
||||
"merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
|
||||
},
|
||||
log_prefix="map.jinja: ",
|
||||
)
|
||||
| load_yaml %}
|
||||
|
||||
{#- Make sure to track `map.jinja` configuration with `_mapdata` #}
|
||||
{%- do default_settings["values"].update(
|
||||
{%- do _formula_settings["values"].update(
|
||||
{
|
||||
"map_jinja": _map_settings["values"]
|
||||
}
|
||||
) %}
|
||||
|
||||
{#- Work around assignment inside for loop #}
|
||||
{#- load configuration values used in `config.get` merging strategies #}
|
||||
{%- set _config = {
|
||||
"stack": default_settings.get("values", {}),
|
||||
"merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
|
||||
"merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
|
||||
} %}
|
||||
|
||||
{#- `parse_matchers` returns a `list` of `dict` with #}
|
||||
{#- - type: `F` to load file, `C`, `G`, `I` for lookup #}
|
||||
{#- - option: specific to the type #}
|
||||
{#- - query: which key is requested #}
|
||||
{#- - query_delimiter: the separator between query component #}
|
||||
{#- - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
|
||||
{#- - value: the result of the `salt[<query_method>](<query>)` #}
|
||||
{%- set map_matchers = parse_matchers(
|
||||
matchers=map_sources,
|
||||
config_get_strategy=_config["merge_strategy"],
|
||||
log_prefix="map.jinja: "
|
||||
)
|
||||
| load_yaml %}
|
||||
|
||||
{%- for matcher in map_matchers %}
|
||||
|
||||
{%- if matcher.type in query_map.keys() %}
|
||||
{#- Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
|
||||
{%- set is_sub_key = matcher.option | default(False) == "SUB" %}
|
||||
|
||||
{#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
|
||||
{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
|
||||
{%- do salt["log.debug"](
|
||||
"map.jinja: merge "
|
||||
~ "sub key " * is_sub_key
|
||||
~ "'"
|
||||
~ matcher.query
|
||||
~ "' retrieved with '"
|
||||
~ matcher.query_method
|
||||
~ "', merge: strategy='"
|
||||
~ _strategy
|
||||
~ "', lists='"
|
||||
~ _config["merge_lists"]
|
||||
~ "'"
|
||||
) %}
|
||||
|
||||
{#- empty value is an empty list, it must be an empty dict for `.update` #}
|
||||
{%- set value = matcher.value | default({}, boolean=True) %}
|
||||
{%- if is_sub_key %}
|
||||
{#- Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
|
||||
{%- set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
|
||||
{%- endif %}
|
||||
{%- do _config.update(
|
||||
{
|
||||
"stack": salt["slsutil.merge"](
|
||||
_config["stack"],
|
||||
value,
|
||||
strategy=_strategy,
|
||||
merge_lists=_config["merge_lists"],
|
||||
)
|
||||
}
|
||||
) %}
|
||||
{%- else %}
|
||||
{#- Load YAML file matching the grain/pillar/... #}
|
||||
{#- Fallback to use the source name as a direct filename #}
|
||||
{#- Mangle `source_key` to use it as literal path #}
|
||||
{%- if matcher.value | length == 0 %}
|
||||
{%- set query_parts = matcher.query.split("/") %}
|
||||
{%- set map_dirname = query_parts[0:-1] | join("/") %}
|
||||
{%- set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
|
||||
{%- else %}
|
||||
{%- set map_dirname = matcher.query %}
|
||||
{%- set map_values = matcher.value %}
|
||||
{%- endif %}
|
||||
|
||||
{#- Some configuration return list #}
|
||||
{%- if map_values is string %}
|
||||
{%- set map_values = [map_values] %}
|
||||
{%- endif %}
|
||||
|
||||
{#- `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
|
||||
{%- set yaml_dir = [
|
||||
map_sources_dir,
|
||||
map_dirname
|
||||
]
|
||||
| select
|
||||
| join("/") %}
|
||||
|
||||
{%- for map_value in map_values %}
|
||||
{%- set yamlfile = [
|
||||
yaml_dir,
|
||||
map_value ~ ".yaml"
|
||||
]
|
||||
| join("/") %}
|
||||
{%- do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
|
||||
{%- load_yaml as loaded_values %}
|
||||
{%- include yamlfile ignore missing %}
|
||||
{%- endload %}
|
||||
|
||||
{%- if loaded_values %}
|
||||
{#- Merge loaded values on the stack #}
|
||||
{%- do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
|
||||
{%- do _config.update(
|
||||
{
|
||||
"stack": salt["slsutil.merge"](
|
||||
_config["stack"],
|
||||
loaded_values.get("values", {}),
|
||||
strategy=loaded_values.get("strategy", "smart"),
|
||||
merge_lists=loaded_values.get("merge_lists", False)
|
||||
| to_bool,
|
||||
)
|
||||
}
|
||||
) %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
|
||||
{%- set mapdata = _config["stack"] %}
|
||||
{%- set mapdata = _formula_settings["values"] %}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user