refactor(map): load formula configuration with libmatchers

openssh/libmapstack.jinja

@@ -3,39 +3,104 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
+
+{%- set _default_config_dirs = [
+      "parameters/",
+      tplroot ~ "/parameters"
+    ] %}
 
 {%- macro mapstack(
-      files,
+      matchers,
       defaults=None,
+      dirs=_default_config_dirs,
       log_prefix="libmapstack: "
     ) %}
 {#-
-    Load YAML files in the order of `files` and merge successively the
+    Load configuration in the order of `matchers` and merge
-    values with `defaults` if the file exists.
+    successively the values with `defaults`.
+
+    The `matchers` are processed using `libmatchers.jinja` to select
+    the configuration sources from where the values are loaded.
 
     Parameters:
-      - `files`: list of files to load
 
-      - `defaults`: default values to start the merging, they are
+      - `matchers`: list of matchers in the form
-        considered built-ins
+        `[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`
+
+      - `defaults`: dictionary of default values to start the merging,
+        they are considered built-ins. It must conform to the same
+        layout as the YAML files: a mandatory `values` key and two
+        optional `strategy` and `merge_lists` keys.
+
+      - `dirs`: list of directory where to look-up the configuration
+        file matching the matchers, by default a global `salt://parameters/`
+        and a per formula `salt://<tplroot>/parameters`
 
       - `log_prefix`: prefix used in the log outputs, by default it is
         `libmapstack: `
 
-    Each YAML file must conform to the following layout:
+    Example: On a Debian system with `roles=["nginx/server", "telegraf"]`
+
+      {%- set settings = mapstack(
+            matchers=[
+              "Y:G@os_family",
+              "I@" ~ tplroot,
+              "Y:C@roles",
+            ],
+            dirs=["defaults", tplroot ~ "/parameters"],
+          )
+          | load_yaml %}
+
+      This will merge the values:
+
+        - starting with the default empty dictionary `{}` (no
+          `defaults` parameter)
+
+        - from the YAML files
+
+          - `salt://defaults/os_family/Debian.yaml`
+
+          - `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`
+
+        - from the pillar `salt["pillar.get"](tplroot)`
+
+        - from the `nginx/server` YAML files:
+
+          - `salt://defaults/roles/nginx/server.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`
+
+        - from the `telegraf` YAML files:
+
+          - `salt://defaults/roles/telegraf.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`
+
+    Each YAML file and the `defaults` parameters must conform to the
+    following layout:
+
       - a mandatory `values` key to store the configuration values
 
       - two optional keys to configure the use of `salt.slsutil.merge`
 
-        - an optional `strategy` key to configure the merging strategy,
+        - an optional `strategy` key to configure the merging
-          for example `strategy: 'recurse'`, the default is `smart`
+          strategy, for example `strategy: 'recurse'`, the default is
+          `smart`
 
-        - an optional `merge_lists` key to configure if lists should be
+        - an optional `merge_lists` key to configure if lists should
-          merged or overridden for the `recurse` and `overwrite`
+          be merged or overridden for the `recurse` and `overwrite`
           strategies, for example `merge_lists: 'true'`
 #}
 {%-   set stack = defaults | default({"values": {} }, boolean=True) %}
 
+{#-   Build configuration file names based on matchers #}
+{%-   set matchers = parse_matchers(
+        matchers,
+        log_prefix=log_prefix
+      )
+      | load_yaml %}
+
 {%-   do salt["log.debug"](
         log_prefix
         ~ "built-in configuration:\n"
@@ -43,28 +108,136 @@
         | yaml(False)
       ) %}
 
-{%-   for yaml_filename in files %}
+{%-   for param_dir in dirs %}
+{%-     for matcher in matchers %}
+{#-       `slsutil.merge` options from #}
+{#-       1. the `value` #}
+{#-       2. the `defaults` #}
+{#-       3. the built-in #}
+{%-       set _strategy = matcher.value
+          | traverse(
+            "strategy",
+            defaults
+            | traverse(
+              "strategy",
+              "smart"
+            )
+          ) %}
+{%-       set _merge_lists = matcher.value
+          | traverse(
+            "merge_lists",
+            defaults
+            | traverse(
+              "merge_lists",
+              False
+            )
+          )
+          | to_bool %}
+
+{%-       if matcher.type in query_map.keys() %}
+{#-         No value is an empty list, must be a dict for `stack.update` #}
+{%-         set normalized_value = matcher.value | default({}, boolean=True) %}
+
+{#-         Merge in `mapdata.<query>` instead of directly in `mapdata` #}
+{%-         set is_sub_key = matcher.option | default(False) == "SUB" %}
+{%-         if is_sub_key %}
+{#-           Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
+{%-           set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
+{%-         else %}
+{%-           set value = normalized_value %}
+{%-         endif %}
+
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "merge "
+              ~ "sub key " * is_sub_key
+              ~ "'"
+              ~ matcher.query
+              ~ "' retrieved with '"
+              ~ matcher.query_method
+              ~ "', merge: strategy='"
+              ~ _strategy
+              ~ "', lists='"
+              ~ _merge_lists
+              ~ "':\n"
+              ~ value
+              | yaml(False)
+            ) %}
+
+{%-         do stack.update(
+              {
+                "values": salt["slsutil.merge"](
+                  stack["values"],
+                  value,
+                  strategy=_strategy,
+                  merge_lists=_merge_lists,
+                )
+              }
+            ) %}
+
+{%-       else %}
+{#-         Load YAML file matching the grain/pillar/... #}
+{#-         Fallback to use the source name as a direct filename #}
+
+{%-         if matcher.value | length == 0 %}
+{#-           Mangle `matcher.value` to use it as literal path #}
+{%-           set query_parts = matcher.query.split("/") %}
+{%-           set yaml_dirname = query_parts[0:-1] | join("/") %}
+{%-           set yaml_names = query_parts[-1] %}
+{%-         else %}
+{%-           set yaml_dirname = matcher.query %}
+{%-           set yaml_names = matcher.value %}
+{%-         endif %}
+
+{#-         Some configuration return list #}
+{%-         if yaml_names is string %}
+{%-           set yaml_names = [yaml_names] %}
+{%-         endif %}
+
+{#-         `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
+{%-         set yaml_dir = [
+              param_dir,
+              yaml_dirname
+            ]
+            | select
+            | join("/") %}
+
+{%-         for yaml_name in yaml_names %}
+{#-           Make sure to have a `.yaml` extension #}
+{#-           Use `.rpartition` to strip last `.yaml` in `dir.yaml/file.yaml` #}
+{%-           set yaml_filename = [
+                yaml_dir.rstrip("/"),
+                yaml_name.rpartition(".yaml")
+                | reject("equalto", ".yaml")
+                | join
+                ~ ".yaml"
+              ]
+              | select
+              | join("/") %}
+
 {%-           do salt["log.debug"](
                 log_prefix
                 ~ "load configuration values from "
                 ~ yaml_filename
               ) %}
-
 {%-           load_yaml as yaml_values %}
 {%-             include yaml_filename ignore missing %}
 {%-           endload %}
 
+{%-           if yaml_values %}
 {%-             do salt["log.debug"](
                   log_prefix
                   ~ "loaded configuration values from "
-          ~ yaml_filename
+                  ~ yaml_name
                   ~ ":\n"
                   ~ yaml_values
                   | yaml(False)
                 ) %}
 
-{%-     if yaml_values %}
+{#-             `slsutil.merge` options from #}
-{#-       Per YAML file `salt["slsutil.merge"]` options or use defaults #}
+{#-             1. the `value` #}
+{#-             2. the `defaults` #}
+{#-             3. the built-in #}
 {%-             set _strategy = yaml_values
                   | traverse(
                     "strategy",
@@ -84,8 +257,6 @@
                     )
                   )
                   | to_bool %}
-
-{#-       Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
 {%-             do stack.update(
                   {
                     "values": salt["slsutil.merge"](
@@ -100,7 +271,7 @@
 {%-             do salt["log.debug"](
                   log_prefix
                   ~ "merged configuration values from "
-            ~ yaml_filename
+                  ~ yaml_name
                   ~ ", merge: strategy='"
                   ~ _strategy
                   ~ "', merge_lists='"
@@ -110,7 +281,9 @@
                   | yaml(False)
                 ) %}
 {%-           endif %}
-
+{%-         endfor %}
+{%-       endif %}
+{%-     endfor %}
 {%-   endfor %}
 
 {%-   do salt["log.debug"](
@@ -121,7 +294,7 @@
       ) %}
 
 {#- Output stack as YAML, caller should use with something like #}
-{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
+{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
 {{ stack | yaml }}
 
 {%- endmacro %}

openssh/map.jinja

@@ -3,11 +3,10 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
-{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
 {%- from tplroot ~ "/libmapstack.jinja" import mapstack %}
 
 {#- Where to lookup parameters source files #}
-{%- set map_sources_dir = tplroot ~ "/parameters" %}
+{%- set formula_param_dir = tplroot ~ "/parameters" %}
 
 {#- List of sources to lookup for parameters #}
 {#- Fallback to previously used grains plus minion `id` #}
@@ -21,13 +20,8 @@
       "Y:G@id",
     ] %}
 
-{#- Allow centralised map.jinja configuration #}
-{%- set _global_param_filename = "parameters/map_jinja.yaml" %}
-{#- Allow per formula map.jinja configuration #}
-{%- set _formula_param_filename = map_sources_dir ~ "/map_jinja.yaml" %}
-
 {%- set _map_settings = mapstack(
-      files=[_global_param_filename, _formula_param_filename],
+      matchers=["map_jinja.yaml"],
       defaults={
         "values": {"sources": map_sources}
       },
@@ -42,136 +36,26 @@
       | yaml(False)
     ) %}
 
-{#- Load formula defaults values #}
+{#- Load formula parameters values #}
-{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
+{%- set _formula_matchers = ["defaults.yaml"] + map_sources %}
-{%- set default_settings = mapstack(
+{%- set _formula_settings = mapstack(
-      files=[_defaults_filename],
+      matchers=_formula_matchers,
-      defaults={"values": {} },
+      dirs=[formula_param_dir],
-      log_prefix="map.jinja defaults: ",
+      defaults={
+        "values": {},
+        "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
+        "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
+      },
+      log_prefix="map.jinja: ",
     )
     | load_yaml %}
+
 {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
-{%- do default_settings["values"].update(
+{%- do _formula_settings["values"].update(
       {
         "map_jinja": _map_settings["values"]
       }
     ) %}
 
-{#- Work around assignment inside for loop #}
-{#- load configuration values used in `config.get` merging strategies #}
-{%- set _config = {
-      "stack": default_settings.get("values", {}),
-      "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
-      "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
-    } %}
-
-{#- `parse_matchers` returns a `list` of `dict` with #}
-{#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
-{#-  - option: specific to the type #}
-{#-  - query: which key is requested #}
-{#-  - query_delimiter: the separator between query component #}
-{#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
-{#-  - value: the result of the `salt[<query_method>](<query>)` #}
-{%- set map_matchers = parse_matchers(
-      matchers=map_sources,
-      config_get_strategy=_config["merge_strategy"],
-      log_prefix="map.jinja: "
-    )
-    | load_yaml %}
-
-{%- for matcher in map_matchers %}
-
-{%-   if matcher.type in query_map.keys() %}
-{#-     Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
-{%-     set is_sub_key = matcher.option | default(False) == "SUB" %}
-
-{#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
-{%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
-{%-     do salt["log.debug"](
-          "map.jinja: merge "
-          ~ "sub key " * is_sub_key
-          ~ "'"
-          ~ matcher.query
-          ~ "' retrieved with '"
-          ~ matcher.query_method
-          ~ "', merge: strategy='"
-          ~ _strategy
-          ~ "', lists='"
-          ~ _config["merge_lists"]
-          ~ "'"
-        ) %}
-
-{#-     empty value is an empty list, it must be an empty dict for `.update` #}
-{%-     set value = matcher.value | default({}, boolean=True) %}
-{%-     if is_sub_key %}
-{#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
-{%-       set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
-{%-     endif %}
-{%-     do _config.update(
-          {
-            "stack": salt["slsutil.merge"](
-              _config["stack"],
-              value,
-              strategy=_strategy,
-              merge_lists=_config["merge_lists"],
-            )
-          }
-        ) %}
-{%-   else %}
-{#-     Load YAML file matching the grain/pillar/... #}
-{#-     Fallback to use the source name as a direct filename #}
-{#-     Mangle `source_key` to use it as literal path #}
-{%-     if matcher.value | length == 0 %}
-{%-       set query_parts = matcher.query.split("/") %}
-{%-       set map_dirname = query_parts[0:-1] | join("/") %}
-{%-       set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
-{%-     else %}
-{%-       set map_dirname = matcher.query %}
-{%-       set map_values = matcher.value %}
-{%-     endif %}
-
-{#-     Some configuration return list #}
-{%-     if map_values is string %}
-{%-       set map_values = [map_values] %}
-{%-     endif %}
-
-{#-     `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
-{%-     set yaml_dir = [
-          map_sources_dir,
-          map_dirname
-        ]
-        | select
-        | join("/") %}
-
-{%-     for map_value in map_values %}
-{%-       set yamlfile = [
-            yaml_dir,
-            map_value ~ ".yaml"
-          ]
-          | join("/") %}
-{%-       do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
-{%-       load_yaml as loaded_values %}
-{%-         include yamlfile ignore missing %}
-{%-       endload %}
-
-{%-       if loaded_values %}
-{#-         Merge loaded values on the stack #}
-{%-         do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
-{%-         do _config.update(
-              {
-                "stack": salt["slsutil.merge"](
-                  _config["stack"],
-                  loaded_values.get("values", {}),
-                  strategy=loaded_values.get("strategy", "smart"),
-                  merge_lists=loaded_values.get("merge_lists", False)
-                  | to_bool,
-                )
-              }
-            ) %}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
-{%- endfor %}
-
 {%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
-{%- set mapdata = _config["stack"] %}
+{%- set mapdata = _formula_settings["values"] %}