refactor(map): load formula configuration with libmatchers

openssh/libmapstack.jinja

@@ -3,39 +3,104 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
+
+{%- set _default_config_dirs = [
+      "parameters/",
+      tplroot ~ "/parameters"
+    ] %}
 
 {%- macro mapstack(
-      files,
+      matchers,
       defaults=None,
+      dirs=_default_config_dirs,
       log_prefix="libmapstack: "
     ) %}
 {#-
-    Load YAML files in the order of `files` and merge successively the
+    Load configuration in the order of `matchers` and merge
-    values with `defaults` if the file exists.
+    successively the values with `defaults`.
+
+    The `matchers` are processed using `libmatchers.jinja` to select
+    the configuration sources from where the values are loaded.
 
     Parameters:
-      - `files`: list of files to load
 
-      - `defaults`: default values to start the merging, they are
+      - `matchers`: list of matchers in the form
-        considered built-ins
+        `[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`
+
+      - `defaults`: dictionary of default values to start the merging,
+        they are considered built-ins. It must conform to the same
+        layout as the YAML files: a mandatory `values` key and two
+        optional `strategy` and `merge_lists` keys.
+
+      - `dirs`: list of directory where to look-up the configuration
+        file matching the matchers, by default a global `salt://parameters/`
+        and a per formula `salt://<tplroot>/parameters`
 
       - `log_prefix`: prefix used in the log outputs, by default it is
         `libmapstack: `
 
-    Each YAML file must conform to the following layout:
+    Example: On a Debian system with `roles=["nginx/server", "telegraf"]`
+
+      {%- set settings = mapstack(
+            matchers=[
+              "Y:G@os_family",
+              "I@" ~ tplroot,
+              "Y:C@roles",
+            ],
+            dirs=["defaults", tplroot ~ "/parameters"],
+          )
+          | load_yaml %}
+
+      This will merge the values:
+
+        - starting with the default empty dictionary `{}` (no
+          `defaults` parameter)
+
+        - from the YAML files
+
+          - `salt://defaults/os_family/Debian.yaml`
+
+          - `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`
+
+        - from the pillar `salt["pillar.get"](tplroot)`
+
+        - from the `nginx/server` YAML files:
+
+          - `salt://defaults/roles/nginx/server.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`
+
+        - from the `telegraf` YAML files:
+
+          - `salt://defaults/roles/telegraf.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`
+
+    Each YAML file and the `defaults` parameters must conform to the
+    following layout:
+
       - a mandatory `values` key to store the configuration values
 
       - two optional keys to configure the use of `salt.slsutil.merge`
 
-        - an optional `strategy` key to configure the merging strategy,
+        - an optional `strategy` key to configure the merging
-          for example `strategy: 'recurse'`, the default is `smart`
+          strategy, for example `strategy: 'recurse'`, the default is
+          `smart`
 
-        - an optional `merge_lists` key to configure if lists should be
+        - an optional `merge_lists` key to configure if lists should
-          merged or overridden for the `recurse` and `overwrite`
+          be merged or overridden for the `recurse` and `overwrite`
           strategies, for example `merge_lists: 'true'`
 #}
 {%-   set stack = defaults | default({"values": {} }, boolean=True) %}
 
+{#-   Build configuration file names based on matchers #}
+{%-   set matchers = parse_matchers(
+        matchers,
+        log_prefix=log_prefix
+      )
+      | load_yaml %}
+
 {%-   do salt["log.debug"](
         log_prefix
         ~ "built-in configuration:\n"
@@ -43,74 +108,182 @@
         | yaml(False)
       ) %}
 
-{%-   for yaml_filename in files %}
+{%-   for param_dir in dirs %}
-{%-     do salt["log.debug"](
+{%-     for matcher in matchers %}
-          log_prefix
+{#-       `slsutil.merge` options from #}
-          ~ "load configuration values from "
+{#-       1. the `value` #}
-          ~ yaml_filename
+{#-       2. the `defaults` #}
-        ) %}
+{#-       3. the built-in #}
-
+{%-       set _strategy = matcher.value
-{%-     load_yaml as yaml_values %}
+          | traverse(
-{%-       include yaml_filename ignore missing %}
+            "strategy",
-{%-     endload %}
+            defaults
-
-{%-     do salt["log.debug"](
-          log_prefix
-          ~ "loaded configuration values from "
-          ~ yaml_filename
-          ~ ":\n"
-          ~ yaml_values
-          | yaml(False)
-        ) %}
-
-{%-     if yaml_values %}
-{#-       Per YAML file `salt["slsutil.merge"]` options or use defaults #}
-{%-       set _strategy = yaml_values
             | traverse(
               "strategy",
-              defaults
+              "smart"
-              | traverse(
+            )
-                "strategy",
+          ) %}
-                "smart"
+{%-       set _merge_lists = matcher.value
-              )
+          | traverse(
-            ) %}
+            "merge_lists",
-{%-       set _merge_lists = yaml_values
+            defaults
             | traverse(
               "merge_lists",
-              defaults
+              False
-              | traverse(
-                "merge_lists",
-                False
-              )
             )
-            | to_bool %}
+          )
+          | to_bool %}
 
-{#-       Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
+{%-       if matcher.type in query_map.keys() %}
-{%-       do stack.update(
+{#-         No value is an empty list, must be a dict for `stack.update` #}
-            {
+{%-         set normalized_value = matcher.value | default({}, boolean=True) %}
-              "values": salt["slsutil.merge"](
-                stack["values"],
-                yaml_values
-                | traverse("values", {}),
-                strategy=_strategy,
-                merge_lists=_merge_lists,
-              )
-            }
-          ) %}
-{%-       do salt["log.debug"](
-            log_prefix
-            ~ "merged configuration values from "
-            ~ yaml_filename
-            ~ ", merge: strategy='"
-            ~ _strategy
-            ~ "', merge_lists='"
-            ~ _merge_lists
-            ~ "':\n"
-            ~ {"values": stack["values"]}
-            | yaml(False)
-          ) %}
-{%-     endif %}
 
+{#-         Merge in `mapdata.<query>` instead of directly in `mapdata` #}
+{%-         set is_sub_key = matcher.option | default(False) == "SUB" %}
+{%-         if is_sub_key %}
+{#-           Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
+{%-           set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
+{%-         else %}
+{%-           set value = normalized_value %}
+{%-         endif %}
+
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "merge "
+              ~ "sub key " * is_sub_key
+              ~ "'"
+              ~ matcher.query
+              ~ "' retrieved with '"
+              ~ matcher.query_method
+              ~ "', merge: strategy='"
+              ~ _strategy
+              ~ "', lists='"
+              ~ _merge_lists
+              ~ "':\n"
+              ~ value
+              | yaml(False)
+            ) %}
+
+{%-         do stack.update(
+              {
+                "values": salt["slsutil.merge"](
+                  stack["values"],
+                  value,
+                  strategy=_strategy,
+                  merge_lists=_merge_lists,
+                )
+              }
+            ) %}
+
+{%-       else %}
+{#-         Load YAML file matching the grain/pillar/... #}
+{#-         Fallback to use the source name as a direct filename #}
+
+{%-         if matcher.value | length == 0 %}
+{#-           Mangle `matcher.value` to use it as literal path #}
+{%-           set query_parts = matcher.query.split("/") %}
+{%-           set yaml_dirname = query_parts[0:-1] | join("/") %}
+{%-           set yaml_names = query_parts[-1] %}
+{%-         else %}
+{%-           set yaml_dirname = matcher.query %}
+{%-           set yaml_names = matcher.value %}
+{%-         endif %}
+
+{#-         Some configuration return list #}
+{%-         if yaml_names is string %}
+{%-           set yaml_names = [yaml_names] %}
+{%-         endif %}
+
+{#-         `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
+{%-         set yaml_dir = [
+              param_dir,
+              yaml_dirname
+            ]
+            | select
+            | join("/") %}
+
+{%-         for yaml_name in yaml_names %}
+{#-           Make sure to have a `.yaml` extension #}
+{#-           Use `.rpartition` to strip last `.yaml` in `dir.yaml/file.yaml` #}
+{%-           set yaml_filename = [
+                yaml_dir.rstrip("/"),
+                yaml_name.rpartition(".yaml")
+                | reject("equalto", ".yaml")
+                | join
+                ~ ".yaml"
+              ]
+              | select
+              | join("/") %}
+
+{%-           do salt["log.debug"](
+                log_prefix
+                ~ "load configuration values from "
+                ~ yaml_filename
+              ) %}
+{%-           load_yaml as yaml_values %}
+{%-             include yaml_filename ignore missing %}
+{%-           endload %}
+
+{%-           if yaml_values %}
+{%-             do salt["log.debug"](
+                  log_prefix
+                  ~ "loaded configuration values from "
+                  ~ yaml_name
+                  ~ ":\n"
+                  ~ yaml_values
+                  | yaml(False)
+                ) %}
+
+{#-             `slsutil.merge` options from #}
+{#-             1. the `value` #}
+{#-             2. the `defaults` #}
+{#-             3. the built-in #}
+{%-             set _strategy = yaml_values
+                  | traverse(
+                    "strategy",
+                    defaults
+                    | traverse(
+                      "strategy",
+                      "smart"
+                    )
+                  ) %}
+{%-             set _merge_lists = yaml_values
+                  | traverse(
+                    "merge_lists",
+                    defaults
+                    | traverse(
+                      "merge_lists",
+                      False
+                    )
+                  )
+                  | to_bool %}
+{%-             do stack.update(
+                  {
+                    "values": salt["slsutil.merge"](
+                      stack["values"],
+                      yaml_values
+                      | traverse("values", {}),
+                      strategy=_strategy,
+                      merge_lists=_merge_lists,
+                    )
+                  }
+                ) %}
+{%-             do salt["log.debug"](
+                  log_prefix
+                  ~ "merged configuration values from "
+                  ~ yaml_name
+                  ~ ", merge: strategy='"
+                  ~ _strategy
+                  ~ "', merge_lists='"
+                  ~ _merge_lists
+                  ~ "':\n"
+                  ~ {"values": stack["values"]}
+                  | yaml(False)
+                ) %}
+{%-           endif %}
+{%-         endfor %}
+{%-       endif %}
+{%-     endfor %}
 {%-   endfor %}
 
 {%-   do salt["log.debug"](
@@ -121,7 +294,7 @@
       ) %}
 
 {#- Output stack as YAML, caller should use with something like #}
-{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
+{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
 {{ stack | yaml }}
 
 {%- endmacro %}

openssh/map.jinja

@@ -3,11 +3,10 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
-{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
 {%- from tplroot ~ "/libmapstack.jinja" import mapstack %}
 
 {#- Where to lookup parameters source files #}
-{%- set map_sources_dir = tplroot ~ "/parameters" %}
+{%- set formula_param_dir = tplroot ~ "/parameters" %}
 
 {#- List of sources to lookup for parameters #}
 {#- Fallback to previously used grains plus minion `id` #}
@@ -21,13 +20,8 @@
       "Y:G@id",
     ] %}
 
-{#- Allow centralised map.jinja configuration #}
-{%- set _global_param_filename = "parameters/map_jinja.yaml" %}
-{#- Allow per formula map.jinja configuration #}
-{%- set _formula_param_filename = map_sources_dir ~ "/map_jinja.yaml" %}
-
 {%- set _map_settings = mapstack(
-      files=[_global_param_filename, _formula_param_filename],
+      matchers=["map_jinja.yaml"],
       defaults={
         "values": {"sources": map_sources}
       },
@@ -42,136 +36,26 @@
       | yaml(False)
     ) %}
 
-{#- Load formula defaults values #}
+{#- Load formula parameters values #}
-{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
+{%- set _formula_matchers = ["defaults.yaml"] + map_sources %}
-{%- set default_settings = mapstack(
+{%- set _formula_settings = mapstack(
-      files=[_defaults_filename],
+      matchers=_formula_matchers,
-      defaults={"values": {} },
+      dirs=[formula_param_dir],
-      log_prefix="map.jinja defaults: ",
+      defaults={
+        "values": {},
+        "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
+        "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
+      },
+      log_prefix="map.jinja: ",
     )
     | load_yaml %}
+
 {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
-{%- do default_settings["values"].update(
+{%- do _formula_settings["values"].update(
       {
         "map_jinja": _map_settings["values"]
       }
     ) %}
 
-{#- Work around assignment inside for loop #}
-{#- load configuration values used in `config.get` merging strategies #}
-{%- set _config = {
-      "stack": default_settings.get("values", {}),
-      "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
-      "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
-    } %}
-
-{#- `parse_matchers` returns a `list` of `dict` with #}
-{#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
-{#-  - option: specific to the type #}
-{#-  - query: which key is requested #}
-{#-  - query_delimiter: the separator between query component #}
-{#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
-{#-  - value: the result of the `salt[<query_method>](<query>)` #}
-{%- set map_matchers = parse_matchers(
-      matchers=map_sources,
-      config_get_strategy=_config["merge_strategy"],
-      log_prefix="map.jinja: "
-    )
-    | load_yaml %}
-
-{%- for matcher in map_matchers %}
-
-{%-   if matcher.type in query_map.keys() %}
-{#-     Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
-{%-     set is_sub_key = matcher.option | default(False) == "SUB" %}
-
-{#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
-{%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
-{%-     do salt["log.debug"](
-          "map.jinja: merge "
-          ~ "sub key " * is_sub_key
-          ~ "'"
-          ~ matcher.query
-          ~ "' retrieved with '"
-          ~ matcher.query_method
-          ~ "', merge: strategy='"
-          ~ _strategy
-          ~ "', lists='"
-          ~ _config["merge_lists"]
-          ~ "'"
-        ) %}
-
-{#-     empty value is an empty list, it must be an empty dict for `.update` #}
-{%-     set value = matcher.value | default({}, boolean=True) %}
-{%-     if is_sub_key %}
-{#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
-{%-       set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
-{%-     endif %}
-{%-     do _config.update(
-          {
-            "stack": salt["slsutil.merge"](
-              _config["stack"],
-              value,
-              strategy=_strategy,
-              merge_lists=_config["merge_lists"],
-            )
-          }
-        ) %}
-{%-   else %}
-{#-     Load YAML file matching the grain/pillar/... #}
-{#-     Fallback to use the source name as a direct filename #}
-{#-     Mangle `source_key` to use it as literal path #}
-{%-     if matcher.value | length == 0 %}
-{%-       set query_parts = matcher.query.split("/") %}
-{%-       set map_dirname = query_parts[0:-1] | join("/") %}
-{%-       set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
-{%-     else %}
-{%-       set map_dirname = matcher.query %}
-{%-       set map_values = matcher.value %}
-{%-     endif %}
-
-{#-     Some configuration return list #}
-{%-     if map_values is string %}
-{%-       set map_values = [map_values] %}
-{%-     endif %}
-
-{#-     `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
-{%-     set yaml_dir = [
-          map_sources_dir,
-          map_dirname
-        ]
-        | select
-        | join("/") %}
-
-{%-     for map_value in map_values %}
-{%-       set yamlfile = [
-            yaml_dir,
-            map_value ~ ".yaml"
-          ]
-          | join("/") %}
-{%-       do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
-{%-       load_yaml as loaded_values %}
-{%-         include yamlfile ignore missing %}
-{%-       endload %}
-
-{%-       if loaded_values %}
-{#-         Merge loaded values on the stack #}
-{%-         do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
-{%-         do _config.update(
-              {
-                "stack": salt["slsutil.merge"](
-                  _config["stack"],
-                  loaded_values.get("values", {}),
-                  strategy=loaded_values.get("strategy", "smart"),
-                  merge_lists=loaded_values.get("merge_lists", False)
-                  | to_bool,
-                )
-              }
-            ) %}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
-{%- endfor %}
-
 {%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
-{%- set mapdata = _config["stack"] %}
+{%- set mapdata = _formula_settings["values"] %}