From 1be0d8725ad933034f4e87cc9636bcc5100bd55c Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:31:22 +0100
Subject: [PATCH 1/8] feat(map): use targeting like syntax for configuration

The `config_get_lookup` and `config_get` sources lack flexibility.

It's not easy to query several pillars and/or grains keys with the
actual system. And the query method is forced to `config.get` without
being configurable by the user.

We define a mechanism to select `map.jinja` sources with similar
notation as the salt targeting system.

The `map.jinja` file uses several sources where to lookup parameter
values. The list of sources can be modified by two files:

1. a global salt://parameters/map_jinja.yaml
2. a per formula salt://{{ tplroot }}/parameters/map_jinja.yaml.

Each source definition has the form `<TYPE>:<OPTION>@<KEY>` where
`<TYPE>` can be one of:

- `Y` to load values from YAML files, this is the default when no type
  is defined
- `C` to lookup values with `config.get`
- `G` to lookup values with `grains.get`
- `I` to lookup values with `pillar.get`

The YAML type option can define the query method to lookup the key
value to build the file name:

- `C` to query with `config.get`, this is the default when to query
  method is defined
- `G` to query with `grains.get`
- `I` to query with `pillar.get`

The `C`, `G` or `I` types can define the `SUB` option to store values
in the sub key `mapdata.<key>` instead of directly in `mapdata`.

Finally, the `<KEY>` describe what to lookup to either build the YAML
filename or gather values using one of the query method.

BREAKING CHANGE: the configuration `map_jinja:sources` is only
                 configurable with `salt://parameters/map_jinja.yaml`
		 and `salt://{{ tplroot }}/parameters/map_jinja.yaml`

BREAKING CHANGE: the `map_jinja:config_get_roots` is replaced by
                 compound like `map_jinja:sources`

BREAKING CHANGE: the two `config_get_lookup` and `config_get` are
                 replaced by `C@<tplroot>:lookup` and `C@<tplroot>`
		 sources
---
 openssh/map.jinja                             | 295 ++++++++++++------
 openssh/parameters/defaults.yaml              |   6 -
 openssh/parameters/map_jinja.yaml             |  21 ++
 .../default/files/_mapdata/amazonlinux-1.yaml |  16 +-
 .../default/files/_mapdata/amazonlinux-2.yaml |  16 +-
 .../files/_mapdata/arch-base-latest.yaml      |  16 +-
 .../default/files/_mapdata/centos-6.yaml      |  16 +-
 .../default/files/_mapdata/centos-7.yaml      |  16 +-
 .../default/files/_mapdata/centos-8.yaml      |  16 +-
 .../default/files/_mapdata/debian-10.yaml     |  16 +-
 .../default/files/_mapdata/debian-9.yaml      |  16 +-
 .../default/files/_mapdata/fedora-31.yaml     |  16 +-
 .../default/files/_mapdata/fedora-32.yaml     |  16 +-
 .../default/files/_mapdata/opensuse-15.yaml   |  16 +-
 .../default/files/_mapdata/ubuntu-16.yaml     |  16 +-
 .../default/files/_mapdata/ubuntu-18.yaml     |  16 +-
 .../default/files/_mapdata/ubuntu-20.yaml     |  16 +-
 17 files changed, 382 insertions(+), 164 deletions(-)
 create mode 100644 openssh/parameters/map_jinja.yaml

diff --git a/openssh/map.jinja b/openssh/map.jinja
index 018e7e1..0363a76 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -8,69 +8,102 @@
 {#- Where to lookup parameters source files #}
 {%- set map_sources_dir = tplroot ~ "/parameters" %}
 
-{#- Load defaults first to allow per formula default map.jinja configuration #}
-{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
-{%- do salt["log.debug"](
-      "map.jinja: initialise parameters from "
-      ~ _defaults_filename
-    ) %}
-{%- import_yaml _defaults_filename as default_settings %}
-
 {#- List of sources to lookup for parameters #}
-{%- do salt["log.debug"]("map.jinja: lookup 'map_jinja' configuration sources") %}
 {#- Fallback to previously used grains plus minion `id` #}
 {%- set map_sources = [
-      "osarch",
-      "os_family",
-      "os",
-      "osfinger",
-      "config_get_lookup",
-      "config_get",
-      "id",
+      "Y:G@osarch",
+      "Y:G@os_family",
+      "Y:G@os",
+      "Y:G@osfinger",
+      "C@" ~ tplroot ~ ":lookup",
+      "C@" ~ tplroot,
+      "Y:G@id",
     ] %}
-{#- Configure map.jinja from defaults.yaml #}
-{%- set map_sources = default_settings | traverse(
-      "values:map_jinja:sources",
-      map_sources,
+{%- do salt["log.debug"](
+      "map.jinja: built-in configuration sources:\n"
+      ~ {"values": {
+          "map_jinja": {"sources": map_sources}
+          }
+        }
+      | yaml(False)
     ) %}
 
-{#- Lookup global sources #}
-{%- set map_sources = salt["config.get"]("map_jinja:sources", map_sources) %}
-{#- Lookup per formula sources #}
-{%- set map_sources = salt["config.get"](
-      tplroot ~ ":map_jinja:sources",
-      map_sources,
+{#- Allow centralised map.jinja configuration #}
+{%- set _global_map_filename = "parameters/map_jinja.yaml" %}
+{%- do salt["log.debug"](
+      "map.jinja: load global map.jinja values from "
+      ~ _global_map_filename
     ) %}
+{%- load_yaml as global_map_settings %}
+{%-   include _global_map_filename ignore missing %}
+{%- endload %}
+
+{%- if global_map_settings %}
+{%-   do salt["log.debug"](
+        "map.jinja: configure sources from global map.jinja configuration "
+        ~ _global_map_filename
+        ~ ":\n"
+        ~ {"map_jinja": global_map_settings}
+        | yaml(False)
+      ) %}
+{%-   set map_sources = global_map_settings
+        | traverse(
+          "values:sources",
+          map_sources,
+      ) %}
+{%- endif %}
+
+{#- Allow per formula map.jinja configuration #}
+{%- set _map_filename = map_sources_dir ~ "/map_jinja.yaml" %}
+{%- do salt["log.debug"](
+      "map.jinja: load per formula map.jinja values from "
+      ~ _map_filename
+    ) %}
+{%- load_yaml as map_settings %}
+{%-   include _map_filename ignore missing %}
+{%- endload %}
+
+{%- if map_settings %}
+{%-   do salt["log.debug"](
+        "map.jinja: configure sources from formula map.jinja configuration "
+        ~ _map_filename
+        ~ ":\n"
+        ~ {"map_jinja": map_settings}
+        | yaml(False)
+      ) %}
+{%-   set map_sources = map_settings
+      | traverse(
+        "values:sources",
+        map_sources,
+      ) %}
+{%- endif %}
 
 {%- do salt["log.debug"](
-      "map.jinja: load parameters with sources from "
+      "map.jinja: load parameters from sources:\n"
       ~ map_sources
+      | yaml(False)
     ) %}
 
-{#- Lookup with `config.get` from configurable roots #}
+{#- Load formula defaults values #}
+{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
 {%- do salt["log.debug"](
-      "map.jinja: initialise 'config.get' roots with 'tplroot' "
-      ~ tplroot
+      "map.jinja: load per formula default values from "
+      ~ _defaults_filename
     ) %}
-{%- set config_get_roots = [tplroot] %}
-{#- Configure `config.get` from defaults.yaml #}
-{%- set config_get_roots = default_settings | traverse(
-      "values:map_jinja:config_get_roots",
-      config_get_roots
-    ) %}
-{#- Lookup global `config.get` roots #}
-{%- set config_get_roots = salt["config.get"](
-      "map_jinja:config_get_roots",
-      config_get_roots
-    ) %}
-{#- Lookup per formula `config.get` roots #}
-{%- set config_get_roots = salt["config.get"](
-      tplroot ~ ":map_jinja:config_get_roots",
-      config_get_roots,
-    ) %}
-{%- do salt["log.debug"](
-      "map.jinja: load parameters with 'config.get' from roots "
-      ~ config_get_roots
+{%- load_yaml as default_settings %}
+{%-   include _defaults_filename ignore missing %}
+{%- endload %}
+
+{%- if not default_settings %}
+{%-   set default_settings = {'values': {} } %}
+{%- endif %}
+
+{#- Make sure to track `map.jinja` configuration with `_mapdata` #}
+{%- do default_settings["values"].update(
+      {
+        "map_jinja": map_settings
+        | traverse("values", {})
+      }
     ) %}
 
 {#- Work around assignment inside for loop #}
@@ -106,63 +139,115 @@
       ) %}
 {%- endif %}
 
+{%- set query_map = {
+      "C": "config.get",
+      "G": "grains.get",
+      "I": "pillar.get",
+    } %}
 
-{#- process each `map.jinja` source #}
+{#- Process each `map.jinja` source #}
+{#- each source has a type: #}
+{#- - `Y` to load values from YAML files (the default when no type is set) #}
+{#- - `C` to lookup values with `config.get` #}
+{#- - `G` to lookup values with `grains.get` #}
+{#- - `I` to lookup values with `pillar.get` #}
+{#- The YAML type option can define query type to build the file name: #}
+{#- - `C` for query with `config.get` (the default when to query type is set) #}
+{#- - `G` for query with `grains.get` #}
+{#- - `I` for query with `pillar.get` #}
+{#- The `C`, `G` or `I` types can define the `SUB` option #}
+{#- to merge values in the sub key `mapdata.<key>` instead of directly in `mapdata` #}
 {%- for map_source in map_sources %}
-{%-   if map_source  in ["config_get", "config_get_lookup"] %}
-{%-     for _config_root in config_get_roots %}
-{%-       set _config_key = {
-            "config_get": _config_root,
-            "config_get_lookup": _config_root ~ ":lookup",
-          }.get(map_source) %}
+{%-   set source_parts = map_source.split('@') %}
+{%-   if source_parts|length == 1 %}
+{#-     By default we load YAML files for config looked up by `config.get` #}
+{%-     set source_type = "Y" %}
+{%-     set query_type = "C" %}
+{%-     set source_key = map_source %}
+{%-   elif source_parts[0][0] == "Y" %}
+{%-     set source_type = "Y" %}
+{%-     set query_type = source_parts[0].split(':')[1] | default("C") %}
+{%-     set source_key = source_parts[1] %}
+{%-   elif source_parts[0][0] in query_map.keys() %}
+{%-     set source_type = source_parts[0].split(':') | first %}
+{%-     set query_type = source_type %}
+{%-     set is_sub_key = source_parts[0].split(':')[1] | default(False) == "SUB" %}
+{%-     set source_key = source_parts[1] %}
+{%-   endif %}
 
-{%-       do salt["log.debug"](
-            "map.jinja: retrieve '"
-            ~ _config_key
-            ~ "' with 'config.get'"
-            ~ _config["merge_msg"]
-          ) %}
-{%-       set _config_get = salt["config.get"](
-            _config_key, default={}, **_config["merge_opt"]
-          ) %}
+{%-   set query_method = query_map[query_type] %}
 
-{#-       `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
-{%-       set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
-{%-       do salt["log.debug"](
-            "map.jinja: merge '"
-            ~ _config_key
-            ~ "' retrieved with 'config.get'"
-            ~ ", merge: strategy='"
-            ~ _strategy
-            ~ "', lists='"
-            ~ _config["merge_lists"]
-            ~ "'"
-          ) %}
+{%-   if source_type in query_map.keys() %}
+{#-     Lookup source `<QUERY_METHOD>@key:to:query` #}
+{%-     if source_type == "C" %}
+{%-       set merge_opts = _config["merge_opt"] %}
+{%-       set merge_msg = _config["merge_msg"] %}
+{%-     else %}
+{#-       No merging strategy supported for `grains.get` and `pillar.get` #}
+{%-       set merge_opts = {} %}
+{%-       set merge_msg = "" %}
+{%-     endif %}
 
-{#-       Keep values under each root key when there are more than one #}
-{%-       if config_get_roots|length > 1 %}
-{%-         set _config_get = { _config_root: _config_get } %}
-{%-       endif %}
-{%-       do _config.update(
-            {
-              "stack": salt["slsutil.merge"](
-                _config["stack"],
-                _config_get,
-                strategy=_strategy,
-                merge_lists=_config["merge_lists"],
-              )
-            }
-          ) %}
-{%-     endfor %}
+{%-     do salt["log.debug"](
+          "map.jinja: retrieve '"
+          ~ source_key
+          ~ "' with '"
+          ~ query_method
+          ~ "'"
+          ~ merge_msg
+        ) %}
+{%-     set _config_get = salt[query_method](
+          source_key,
+          default={},
+          **merge_opts
+        ) %}
+
+{#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
+{%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
+{%-     do salt["log.debug"](
+          "map.jinja: merge "
+          ~ "sub key " * is_sub_key
+          ~ "'"
+          ~ source_key
+          ~ "' retrieved with '"
+          ~ query_method
+          ~ "', merge: strategy='"
+          ~ _strategy
+          ~ "', lists='"
+          ~ _config["merge_lists"]
+          ~ "'"
+        ) %}
+
+{%-     if is_sub_key %}
+{#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
+{%-       set _config_get = { source_key.rstrip(':lookup'): _config_get } %}
+{%-     endif %}
+{%-     do _config.update(
+          {
+            "stack": salt["slsutil.merge"](
+              _config["stack"],
+              _config_get,
+              strategy=_strategy,
+              merge_lists=_config["merge_lists"],
+            )
+          }
+        ) %}
 {%-   else %}
-{#-     Lookup the grain/pillar/... #}
+{#-     Load YAML file matching the grain/pillar/... #}
 {#-     Fallback to use the source name as a direct filename #}
-{%-     set map_values = salt["config.get"](map_source, []) %}
+{%-     do salt["log.debug"](
+          "map.jinja: lookup '"
+          ~ source_key
+          ~ "' with '"
+          ~ query_method
+          ~ "'"
+        ) %}
+{%-     set map_values = salt[query_method](source_key, []) %}
 
-{#-     Mangle `map_source` to use it as literal path #}
+{#-     Mangle `source_key` to use it as literal path #}
 {%-     if map_values | length == 0 %}
-{%-       set map_source_parts = map_source.split("/") %}
-{%-       set map_source = map_source_parts[0:-1] | join("/") %}
+{%-       set map_source_parts = source_key.split("/") %}
+{%-       set source_key = map_source_parts[0:-1] | join("/") %}
 {%-       set map_values = map_source_parts[-1].rstrip(".yaml") %}
 {%-     endif %}
 
@@ -171,14 +256,20 @@
 {%-       set map_values = [map_values] %}
 {%-     endif %}
 
+{#-     `source_key` can be an empty string with literal path like `myconf.yaml` #}
+{%-     set yaml_dir = [
+          map_sources_dir,
+          source_key
+        ]
+        | select
+        | join("/") %}
+
 {%-     for map_value in map_values %}
 {%-       set yamlfile = [
-            map_sources_dir,
-            map_source,
-            map_value ~ ".yaml",
+            yaml_dir,
+            map_value ~ ".yaml"
           ]
-          | join("/")
-          %}
+          | join("/") %}
 {%-       do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
 {%-       load_yaml as loaded_values %}
 {%-         include yamlfile ignore missing %}
diff --git a/openssh/parameters/defaults.yaml b/openssh/parameters/defaults.yaml
index 2fe08d4..b7c4436 100644
--- a/openssh/parameters/defaults.yaml
+++ b/openssh/parameters/defaults.yaml
@@ -2,12 +2,6 @@
 # vim: ft=yaml
 ---
 values:
-  map_jinja:
-    config_get_roots:
-      - openssh
-      - sshd_config
-      - ssh_config
-
   openssh:
     sshd_enable: true
     sshd_binary: /usr/sbin/sshd
diff --git a/openssh/parameters/map_jinja.yaml b/openssh/parameters/map_jinja.yaml
new file mode 100644
index 0000000..7d479b0
--- /dev/null
+++ b/openssh/parameters/map_jinja.yaml
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+values:
+  sources:
+    - "Y:G@osarch"
+    - "Y:G@os_family"
+    - "Y:G@os"
+    - "Y:G@osfinger"
+
+    # Merge values from `config.get` under `mapdata.<key>` to keep
+    # compatibility with user pillars.
+    # The `<key>` and `<key>:lookup` are merged together
+    - "C:SUB@openssh:lookup"
+    - "C:SUB@openssh"
+    - "C:SUB@sshd_config:lookup"
+    - "C:SUB@sshd_config"
+    - "C:SUB@ssh_config:lookup"
+    - "C:SUB@ssh_config"
+
+    - "Y:G@id"
diff --git a/test/integration/default/files/_mapdata/amazonlinux-1.yaml b/test/integration/default/files/_mapdata/amazonlinux-1.yaml
index c4ed9c9..5a1098f 100644
--- a/test/integration/default/files/_mapdata/amazonlinux-1.yaml
+++ b/test/integration/default/files/_mapdata/amazonlinux-1.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/amazonlinux-2.yaml b/test/integration/default/files/_mapdata/amazonlinux-2.yaml
index 5311f2c..605165d 100644
--- a/test/integration/default/files/_mapdata/amazonlinux-2.yaml
+++ b/test/integration/default/files/_mapdata/amazonlinux-2.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/arch-base-latest.yaml b/test/integration/default/files/_mapdata/arch-base-latest.yaml
index 8f62a0e..a3fffb5 100644
--- a/test/integration/default/files/_mapdata/arch-base-latest.yaml
+++ b/test/integration/default/files/_mapdata/arch-base-latest.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/centos-6.yaml b/test/integration/default/files/_mapdata/centos-6.yaml
index 1f88c5f..26145db 100644
--- a/test/integration/default/files/_mapdata/centos-6.yaml
+++ b/test/integration/default/files/_mapdata/centos-6.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/centos-7.yaml b/test/integration/default/files/_mapdata/centos-7.yaml
index dffc0a7..63eac30 100644
--- a/test/integration/default/files/_mapdata/centos-7.yaml
+++ b/test/integration/default/files/_mapdata/centos-7.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/centos-8.yaml b/test/integration/default/files/_mapdata/centos-8.yaml
index d3ea759..991ad4b 100644
--- a/test/integration/default/files/_mapdata/centos-8.yaml
+++ b/test/integration/default/files/_mapdata/centos-8.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/debian-10.yaml b/test/integration/default/files/_mapdata/debian-10.yaml
index bc3bcc0..b41f767 100644
--- a/test/integration/default/files/_mapdata/debian-10.yaml
+++ b/test/integration/default/files/_mapdata/debian-10.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/debian-9.yaml b/test/integration/default/files/_mapdata/debian-9.yaml
index 185dae0..3f5ed73 100644
--- a/test/integration/default/files/_mapdata/debian-9.yaml
+++ b/test/integration/default/files/_mapdata/debian-9.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/fedora-31.yaml b/test/integration/default/files/_mapdata/fedora-31.yaml
index 53bd782..a8ee627 100644
--- a/test/integration/default/files/_mapdata/fedora-31.yaml
+++ b/test/integration/default/files/_mapdata/fedora-31.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/fedora-32.yaml b/test/integration/default/files/_mapdata/fedora-32.yaml
index eedcbd5..4088e15 100644
--- a/test/integration/default/files/_mapdata/fedora-32.yaml
+++ b/test/integration/default/files/_mapdata/fedora-32.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/opensuse-15.yaml b/test/integration/default/files/_mapdata/opensuse-15.yaml
index 5c7f1b1..f4a741c 100644
--- a/test/integration/default/files/_mapdata/opensuse-15.yaml
+++ b/test/integration/default/files/_mapdata/opensuse-15.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/ubuntu-16.yaml b/test/integration/default/files/_mapdata/ubuntu-16.yaml
index 268bc13..aff89d2 100644
--- a/test/integration/default/files/_mapdata/ubuntu-16.yaml
+++ b/test/integration/default/files/_mapdata/ubuntu-16.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/ubuntu-18.yaml b/test/integration/default/files/_mapdata/ubuntu-18.yaml
index 9330166..f592970 100644
--- a/test/integration/default/files/_mapdata/ubuntu-18.yaml
+++ b/test/integration/default/files/_mapdata/ubuntu-18.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false
diff --git a/test/integration/default/files/_mapdata/ubuntu-20.yaml b/test/integration/default/files/_mapdata/ubuntu-20.yaml
index 948b878..93a2a39 100644
--- a/test/integration/default/files/_mapdata/ubuntu-20.yaml
+++ b/test/integration/default/files/_mapdata/ubuntu-20.yaml
@@ -3,10 +3,18 @@
 ---
 values:
   map_jinja:
-    config_get_roots:
-    - openssh
-    - sshd_config
-    - ssh_config
+    sources:
+    - Y:G@osarch
+    - Y:G@os_family
+    - Y:G@os
+    - Y:G@osfinger
+    - C:SUB@openssh:lookup
+    - C:SUB@openssh
+    - C:SUB@sshd_config:lookup
+    - C:SUB@sshd_config
+    - C:SUB@ssh_config:lookup
+    - C:SUB@ssh_config
+    - Y:G@id
   openssh:
     absent_dsa_keys: false
     absent_ecdsa_keys: false

From 7ecb24bdc1ff84ddac4c7c3e5d8d70c7512f4fb5 Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:33:25 +0100
Subject: [PATCH 2/8] docs(map): document the new `map.jinja` with targeting
 like syntax

---
 docs/README.rst    |   6 +
 docs/map.jinja.rst | 447 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 453 insertions(+)
 create mode 100644 docs/map.jinja.rst

diff --git a/docs/README.rst b/docs/README.rst
index d2f45a4..5f65482 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -31,6 +31,12 @@ which contains the currently released version. This formula is versioned accordi
 
 See `Formula Versioning Section <https://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html#versioning>`_ for more details.
 
+If you need (non-default) configuration, please refer to:
+
+- `how to configure the formula with map.jinja <map.jinja.rst>`_
+- the ``pillar.example`` file
+
+
 Contributing to this repo
 -------------------------
 
diff --git a/docs/map.jinja.rst b/docs/map.jinja.rst
new file mode 100644
index 0000000..dab217b
--- /dev/null
+++ b/docs/map.jinja.rst
@@ -0,0 +1,447 @@
+.. _map.jinja:
+
+``map.jinja``: gather formula configuration values
+==================================================
+
+The `documentation`_ explains the use of a ``map.jinja`` to gather parameters values for a formula.
+
+As `pillars`_ are rendered on the Salt master for every minion, this increases the load on the master as the pillar values and the number of minions grows.
+
+As a good practice, you should:
+
+- store non-secret data in YAML files distributed by the `fileserver`_
+- store secret data in:
+
+  - `pillars`_ (and look for the use of something like `pillar.vault`_)
+  - `SDB`_ (and look for the use of something like `sdb.vault`_)
+
+Current best practice is to let ``map.jinja`` handle parameters from all sources, to minimise the use of pillars, grains or configuration from ``sls`` files and templates directly.
+
+
+.. contents:: **Table of Contents**
+
+
+For formula users
+-----------------
+
+
+Quick start: configure per role and per DNS domain name values
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We will see a quick setup to configure the ``TEMPLATE`` formula for different DNS domain name and several roles.
+
+For this example, I'll define 2 kinds of `fileserver`_ sources:
+
+1. formulas git repositories with hard-coded version reference to avoid breaking my setup randomly at upstream update. they are the last sources where files are looked up
+2. parameters of the formulas in the file backend `roots`_
+
+
+Configure the fileserver backends
+`````````````````````````````````
+
+I configure the `fileserver`_ backends to serve:
+
+1. files from `roots`_ first
+2. `gitfs`_ repositories last
+
+Create the file ``/etc/salt/master.d/fileserver.conf`` and restart the ``master``:
+
+.. code-block:: yaml
+
+    ---
+    ##
+    ## file server
+    ##
+    fileserver_backend:
+      # parameters values and override
+      - roots
+      # formulas
+      - gitfs
+
+    # The files in this directory will take precedence over git repositories
+    file_roots:
+      base:
+        - /srv/salt
+
+    # List of formulas I'm using
+    gitfs_remotes:
+      - https://github.com/saltstack-formulas/template-formula.git:
+        - base: v4.1.1
+      - https://github.com/saltstack-formulas/openssh-formula.git:
+        - base: v2.0.1
+    ...
+
+
+Create per DNS configuration for ``TEMPLATE`` formula
+`````````````````````````````````````````````````````
+
+Now, we can provides the per DNS domain name configuration files for the ``TEMPLATE`` formulas under ``/srv/salt/TEMPLATE/parameters/``.
+
+We create the directory for ``dns:domain`` grain and we add a symlink for the ``domain`` grain which is extracted from the minion ``id``:
+
+.. code-block:: console
+
+    mkdir -p /srv/salt/TEMPLATE/parameters/dns:domain/
+    ln -s dns:domain /srv/salt/TEMPLATE/parameters/domain
+
+We create a configuration for the DNS domain ``example.net`` in ``/srv/salt/TEMPLATE/parameters/dns:domain/example.net.yaml``:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      config: /etc/template-formula-example-net.conf
+    ...
+
+We create another configuration for the DNS domain ``example.com`` in ``/srv/salt/TEMPLATE/parameters/dns:domain/example.com.yaml``:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      config: /etc/template-formula-{{ grains['os_family'] }}.conf
+    ...
+
+
+Create per role configuration for ``TEMPLATE`` formula
+``````````````````````````````````````````````````````
+
+Now, we can provides the per role configuration files for the ``TEMPLATE`` formulas under ``/srv/salt/TEMPLATE/parameters/``.
+
+We create the directory for roles:
+
+.. code-block:: console
+
+    mkdir -p /srv/salt/TEMPLATE/parameters/roles
+
+We will define 2 roles:
+
+- ``TEMPLATE/server``
+- ``TEMPLATE/client``
+
+We create a configuration for the role ``TEMPLATE/server`` in ``/srv/salt/TEMPLATE/parameters/roles/TEMPLATE/server.yaml``:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      config: /etc/template-formula-server.conf
+    ...
+
+We create another configuration for the role ``TEMPLATE/client`` in ``/srv/salt/TEMPLATE/parameters/roles/TEMPLATE/client.yaml``:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      config: /etc/template-formula-client.conf
+    ...
+
+
+Enable roles and the ``dns:domain`` and ``domain`` grains for ``map.jinja``
+```````````````````````````````````````````````````````````````````````````
+
+We need to redefine the sources for ``map.jinja`` to load values from our new configuration files, we provide a global configuration for all our minions.
+
+We create the global parameters file ``/srv/salt/parameters/map_jinja.yaml``:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      sources:
+        # default values
+        - "Y:G@osarch"
+        - "Y:G@os_family"
+        - "Y:G@os"
+        - "Y:G@osfinger"
+        - "C@{{ tplroot ~ ':lookup' }}"
+        - "C@{{ tplroot }}"
+
+        # Roles activate/deactivate things
+        # then thing are configured depending on environment
+        # So roles comes before `dns:domain`, `domain` and `id`
+        - "Y:C@roles"
+
+        # DNS domain configured (DHCP or resolv.conf)
+        - "Y:G@dns:domain"
+
+        # Based on minion ID
+        - "Y:G@domain"
+
+        # default values
+        - "Y:G@id"
+    ...
+
+The syntax is explained later at `Sources of configuration values`_.
+
+
+Bind roles to minions
+`````````````````````
+
+We associate roles `grains`_ to minion using `grains.append`_.
+
+For the servers:
+
+.. code-block:: console
+
+    salt 'server-*' grains.append roles TEMPLATE/server
+
+For the clients:
+
+.. code-block:: console
+
+    salt 'client-*' grains.append roles TEMPLATE/client
+
+.. note::
+
+    Since we used ``Y:C@roles``, ``map.jinja`` will do a ``salt['config.get']('roles')`` to retrieve the roles so you could use any other method to bind roles to minions (`pillars`_ or `SDB`_) but `grains`_ seems to be the prefered method.
+
+
+Format of configuration YAML files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you write a new YAML file, note that it must conform to the following layout:
+
+- a mandatory ``values`` key to store the configuration values
+- two optional keys to configure the use of `salt.slsutil.merge`_
+
+  - an optional ``strategy`` key to configure the merging strategy, for example ``strategy: 'recurse'``, the default is ``smart``
+  - an optional ``merge_lists`` key to configure if lists should be merged or overridden for the ``recurse`` and ``overwrite`` strategy, for example ``merge_lists: 'true'``
+
+Here is a valid example:
+
+.. code-block:: yaml
+
+    ---
+    strategy: 'recurse'
+    merge_lists: 'false'
+    values:
+      pkg:
+        name: 'some-package'
+      config: '/path/to/a/configuration/file'
+    ...
+
+You can use `Jinja`_ as with any SLS files:
+
+.. code-block:: yaml
+
+    ---
+    strategy: 'overwrite'
+    merge_lists: 'true'
+    values:
+      output_dir: /tmp/{{ grains['id'] }}
+    ...
+
+
+Sources of configuration values
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+Configuring ``map.jinja`` sources
+`````````````````````````````````
+
+The ``map.jinja`` file uses several sources where to lookup parameter values. The list of sources can be modified by two files:
+
+1. a global ``salt://parameters/map_jinja.yaml``
+2. a per formula ``salt://{{ tplroot }}/parameters/map_jinja.yaml``, it overrides the global configuration
+
+Each source definition has the form ``<TYPE>:<OPTION>@<KEY>`` where ``<TYPE>`` can be one of:
+
+- ``Y`` to load values from YAML files from the `fileserver`_, this is the default when no type is defined
+- ``C`` to lookup values with `salt['config.get']`_
+- ``G`` to lookup values with `salt['grains.get']`_
+- ``I`` to lookup values with `salt['pillar.get']`_
+
+The YAML type option can define the query method to lookup the key value to build the file name:
+
+- ``C`` to query with `salt['config.get']`_, this is the default when no query method is defined
+- ``G`` to query with `salt['grains.get']`_
+- ``I`` to query with `salt['pillar.get']`_
+
+The ``C``, ``G`` or ``I`` types can define the ``SUB`` option to store values in the sub key ``mapdata.<KEY>`` instead of directly in ``mapdata``.
+
+Finally, the ``<KEY>`` describes what to lookup to either build the YAML filename or gather values using one of the query methods.
+
+.. note::
+
+    For the YAML type, if the ``<KEY>`` can't be looked up, then it's used a literal string path to a YAML file, for example: ``any/path/can/be/used/here.yaml`` will result in the loading of ``salt://{{ tplroot }}/parameters/any/path/can/be/used/here.yaml`` if it exists.
+
+The built-in ``map_jinja:sources`` is:
+
+.. code-block:: yaml
+
+    - "Y:G@osarch"
+    - "Y:G@os_family"
+    - "Y:G@os"
+    - "Y:G@osfinger"
+    - "C@{{ tplroot ~ ':lookup' }}"
+    - "C@{{ tplroot }}"
+    - "Y:G@id"
+
+This is strictly equivalent to the following ``map_jinja.yaml`` using `Jinja`_:
+
+.. code-block:: sls
+
+    values:
+      sources:
+        - "parameters/osarch/{{ salt['grains.get']('osarch') }}.yaml"
+        - "parameters/os_family/{{ salt['grains.get']('os_family') }}.yaml"
+        - "parameters/os/{{ salt['grains.get']('os') }}.yaml"
+        - "parameters/osfinger/{{ salt['grains.get']('osfinger') }}.yaml"
+        - "C@{{ tplroot ~ ':lookup' }}"
+        - "C@{{ tplroot }}"
+        - "parameters/id/{{ salt['grains.get']('id') }}.yaml"
+
+
+Loading values from the configuration sources
+`````````````````````````````````````````````
+
+For each configuration source defined in ``map_jinja:sources``, ``map.jinja`` will:
+
+#. load values depending on the source type:
+
+   - for YAML file sources
+
+     - if the ``<KEY>`` can be looked up, load values from the YAML file named ``salt://{{ tplroot }}/paramaters/<KEY>/{{ salt['<QUERY_METHOD>']('<KEY>') }}.yaml`` if it exists
+     - otherwise, load the YAML file named ``salt://{{ tplroot }}/parameters/<KEY>.yaml`` if it exists
+
+   - for ``C``, ``G`` or ``I`` source type, lookup the value of ``salt['<QUERY_METHOD>']('<KEY>')``
+
+#. merge the loaded values with the previous ones using `salt.slsutil.merge`_
+
+There will be no error if a YAML file does not exists, they are all optional.
+
+
+Configuration values from ``salt['config.get']``
+````````````````````````````````````````````````
+
+For sources with of type ``C`` declared in ``map_jinja:sources``, you can configure the ``merge`` option of `salt['config.get']`_ by defining per formula ``strategy`` configuration key (retrieved with ``salt['config.get'](tplroot ~ ':strategy')`` with one of the following values:
+
+- ``recurse`` merge recursively dictionaries. Non dictionary values replace already defined values
+- ``overwrite`` new value completely replace old ones
+
+By default, no merging is done, the first value found is returned.
+
+
+Global view of the order of preferences
+```````````````````````````````````````
+
+To summarize, here is a complete example of the load order of formula configuration values for an ``AMD64`` ``Ubuntu 18.04`` minion named ``minion1.example.net`` for the ``libvirt`` formula:
+
+#. ``parameters/defaults.yaml``
+#. ``parameters/osarch/amd64.yaml``
+#. ``parameters/os_family/Debian.yaml``
+#. ``parameters/os/Ubuntu.yaml``
+#. ``parameters/osfinger/Ubuntu-18.04.yaml``
+#. ``salt['config.get']('libvirt:lookup')``
+#. ``salt['config.get']('libvirt')``
+#. ``parameters/id/minion1.example.net``
+
+Remember that the order is important, for example, the value of ``key1:subkey1`` loaded from ``parameters/os_family/Debian.yaml`` is overridden by a value loaded from ``parameters/id/minion1.example.net``.
+
+
+For formula authors and contributors
+------------------------------------
+
+Dependencies
+^^^^^^^^^^^^
+
+``map.jinja`` requires:
+
+- salt minion 2018.3.3 minimum to use the `traverse`_ jinja filter
+- to be located at the root of the formula named directory (e.g. ``libvirt-formula/libvirt/map.jinja``)
+- the ``libsaltcli.jinja`` library, stored in the same directory, to disable the ``merge`` option of `salt['config.get']`_ over `salt-ssh`_
+
+
+Use formula configuration values in ``sls``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``map.jinja`` exports a unique ``mapdata`` variable which could be renamed during import.
+
+Here is the best way to use it in an ``sls`` file:
+
+.. code-block:: sls
+
+    {#- Get the `tplroot` from `tpldir` #}
+    {%- set tplroot = tpldir.split("/")[0] %}
+    {%- from tplroot ~ "/map.jinja" import mapdata as TEMPLATE with context %}
+
+    test-does-nothing-but-display-TEMPLATE-as-json:
+      test.nop:
+        - name: {{ TEMPLATE | json }}
+
+
+Use formula configuration values in templates
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you need to process salt templates, you should avoid calling `salt['config.get']`_ (or `salt['pillar.get']`_ and `salt['grains.get']`_) directly from the template. All the needed values should be available within the ``mapdata`` variable exported by ``map.jinja``.
+
+Here is an example based on `template-formula/TEMPLATE/config/file.sls`_:
+
+.. code-block:: sls
+
+    # -*- coding: utf-8 -*-
+    # vim: ft=sls
+
+    {#- Get the `tplroot` from `tpldir` #}
+    {%- set tplroot = tpldir.split('/')[0] %}
+    {%- set sls_package_install = tplroot ~ '.package.install' %}
+    {%- from tplroot ~ "/map.jinja" import mapdata as TEMPLATE with context %}
+    {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
+
+    include:
+      - {{ sls_package_install }}
+
+    TEMPLATE-config-file-file-managed:
+      file.managed:
+        - name: {{ TEMPLATE.config }}
+        - source: {{ files_switch(['example.tmpl'],
+                                  lookup='TEMPLATE-config-file-file-managed'
+                     )
+                  }}
+        - mode: 644
+        - user: root
+        - group: {{ TEMPLATE.rootgroup }}
+        - makedirs: True
+        - template: jinja
+        - require:
+          - sls: {{ sls_package_install }}
+        - context:
+            TEMPLATE: {{ TEMPLATE | json }}
+
+This ``sls`` file expose a ``TEMPLATE`` context variable to the jinja template which could be used like this:
+
+.. code-block:: jinja
+
+    ########################################################################
+    # File managed by Salt at <{{ source }}>.
+    # Your changes will be overwritten.
+    ########################################################################
+
+    This is another example file from SaltStack template-formula.
+
+    # This is here for testing purposes
+    {{ TEMPLATE | json }}
+
+    winner of the merge: {{ TEMPLATE['winner'] }}
+
+
+.. _documentation: https://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html#writing-formulas
+.. _fileserver: https://docs.saltstack.com/en/latest/ref/file_server
+.. _salt['config.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.config.html#salt.modules.config.get
+.. _salt['grains.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.grains.html#salt.modules.grains.get
+.. _salt['pillar.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.pillar.html#salt.modules.pillar.get
+.. _pillar.vault: https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.vault.html
+.. _pillars: https://docs.saltstack.com/en/latest/topics/pillar/
+.. _grains: https://docs.saltstack.com/en/latest/topics/grains/
+.. _grains.append: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.grains.html#salt.modules.grains.append
+.. _SDB: https://docs.saltstack.com/en/latest/topics/sdb/index.html
+.. _sdb.vault: https://docs.saltstack.com/en/latest/ref/sdb/all/salt.sdb.vault.html
+.. _Jinja: https://docs.saltstack.com/en/latest/topics/jinja
+.. _roots: https://docs.saltstack.com/en/latest/ref/file_server/all/salt.fileserver.roots.html
+.. _gitfs: https://docs.saltstack.com/en/latest/topics/tutorials/gitfs.html
+.. _salt.slsutil.merge: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.slsutil.html
+.. _traverse: https://docs.saltstack.com/en/latest/topics/jinja/index.html#traverse
+.. _salt-ssh: https://docs.saltstack.com/en/latest/topics/ssh/
+.. _template-formula/TEMPLATE/config/file.sls: https://github.com/saltstack-formulas/template-formula/blob/master/TEMPLATE/config/file.sls

From 925c86ea698c68f684ba1645a58c88d688e6acc5 Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:41:20 +0100
Subject: [PATCH 3/8] refactor(map): compound matchers like parsing with
 `libmatchers`

---
 openssh/libmatchers.jinja | 208 ++++++++++++++++++++++++++++++++++++++
 openssh/map.jinja         | 137 ++++++-------------------
 2 files changed, 240 insertions(+), 105 deletions(-)
 create mode 100644 openssh/libmatchers.jinja

diff --git a/openssh/libmatchers.jinja b/openssh/libmatchers.jinja
new file mode 100644
index 0000000..b5e768d
--- /dev/null
+++ b/openssh/libmatchers.jinja
@@ -0,0 +1,208 @@
+{#- -*- coding: utf-8 -*- #}
+{#- vim: ft=jinja #}
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libsaltcli.jinja" import cli %}
+
+{%- set query_map = {
+      "C": "config.get",
+      "G": "grains.get",
+      "I": "pillar.get",
+    } %}
+
+{#- When no part before `@` is provided: #}
+{#-   - define a filename path, noted `F` #}
+{#-   - use `salt["config.get"]`, noted `C` #}
+{%- set _defaults = {
+      "type": "F",
+      "query_type": "C",
+    } %}
+
+{%- macro parse_matchers(
+      matchers,
+      config_get_strategy=None,
+      log_prefix="libmatchers: "
+    ) %}
+{#-   matcher format is `[<TYPE>[:<OPTION>]@]<KEY>` #}
+{#-   each matcher has a type: #}
+{#-   - `F` to build a file name (the default when no type is set) #}
+{#-   - `C` to lookup values with `config.get` #}
+{#-   - `G` to lookup values with `grains.get` #}
+{#-   - `I` to lookup values with `pillar.get` #}
+{#-   The `FILE` type option can define query type to build the file name: #}
+{#-   - `C` for query with `config.get` (the default when to query type is set) #}
+{#-   - `G` for query with `grains.get` #}
+{#-   - `I` for query with `pillar.get` #}
+{%-   set parsed_matchers = [] %}
+{%-   for matcher in matchers %}
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "process matcher: '"
+          ~ matcher
+          ~ "'"
+        ) %}
+
+{%-     set parsed = {} %}
+{%-     set matcher_parts = matcher.split('@') %}
+{%-     if matcher_parts | length == 1 %}
+{#-       By default we load YAML files for config looked up by `config.get` #}
+{%-       do parsed.update(
+            {
+              "type": _defaults["type"],
+              "option": None,
+              "query_method": query_map[_defaults["query_type"]],
+              "query": matcher
+            }
+          ) %}
+{%-       do salt["log.debug"](
+            log_prefix
+            ~ "use built-in defaults for matcher:\n"
+            ~ parsed
+            | yaml(False)
+            | indent(4, True)
+          ) %}
+{%-     else %}
+{%-       do salt["log.debug"](
+            log_prefix
+            ~ "parse matcher: '"
+            ~ matcher
+            ~ "'"
+          ) %}
+{%-       set metadatas = matcher_parts[0].split(":") %}
+{%-       do parsed.update(
+            {
+              "query": matcher_parts[1]
+            }
+          ) %}
+{%-       if metadatas | length == 1 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": "C",
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 1 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 2 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 2 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 3 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 3 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 4 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 4 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       endif %}
+{%-     endif %}
+
+{#-     The `<OPTION>` has different meaning based on type #}
+{%-     if query_map.get(parsed.type, False) %}
+{%-       do parsed.update(
+            {
+              "query_method": query_map[parsed.type]
+            }
+          ) %}
+{%-     else %}
+{%-       do parsed.update(
+            {
+              "query_method": query_map[
+                parsed.option
+                | default("C", boolean=True)
+              ]
+            }
+          ) %}
+{%-     endif %}
+
+{#-     Add `merge:` option to `salt["config.get"]` if configured #}
+{%-     if cli in ["minion", "local"] and parsed.query_method == "config.get" and config_get_strategy %}
+{%-       set merge_opt = {"merge": config_get_strategy} %}
+{%-       set merge_msg = (
+            ", merge: strategy='"
+            ~ config_get_strategy
+            ~ "'"
+          ) %}
+{%-     else %}
+{%-       if cli not in ["minion", "local"] %}
+{%-         do salt["log.error"](
+              log_prefix
+              ~ "the 'merge' option of 'config.get' is skipped when the salt command type is '"
+              ~ cli
+              ~ "'"
+            ) %}
+{%-       endif %}
+{%-       set merge_opt = {} %}
+{%-       set merge_msg = "" %}
+{%-     endif %}
+
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "lookup '"
+          ~ parsed.query
+          ~ "' with '"
+          ~ parsed.query_method
+          ~ "'"
+          ~ merge_msg
+        ) %}
+{%-     set values = salt[parsed.query_method](
+          parsed.query,
+          default=[],
+          **merge_opt
+        ) %}
+{%-     do parsed.update(
+          {
+            "value": values
+          }
+        ) %}
+
+{%-     do parsed_matchers.append(parsed) %}
+
+{%-   endfor %}
+{%-   do salt["log.debug"](
+        log_prefix
+        ~ "parsed matchers:\n"
+        ~ parsed_matchers
+        | yaml(False)
+        | indent(4, True)
+      ) %}
+
+{{ parsed_matchers | yaml }}
+{%- endmacro %}
diff --git a/openssh/map.jinja b/openssh/map.jinja
index 0363a76..a12d3ea 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -3,7 +3,7 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
-{%- from tplroot ~ "/libsaltcli.jinja" import cli with context %}
+{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
 
 {#- Where to lookup parameters source files #}
 {%- set map_sources_dir = tplroot ~ "/parameters" %}
@@ -114,93 +114,24 @@
       "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
     } %}
 
-{#- the `config.get` merge option only works for `minion` or `local` salt command types #}
-{%- if cli in ["minion", "local"] %}
-{%-   do _config.update(
-        {
-          "merge_opt": {"merge": _config["merge_strategy"]},
-          "merge_msg": ", merge: strategy='" ~ _config["merge_strategy"] ~ "'",
-        }
-      ) %}
-{#- the `config.get` merge option is not available for `ssh` or `unknown` salt command types #}
-{%- else %}
-{%-   if _config["merge_strategy"] %}
-{%-     do salt["log.error"](
-          "map.jinja: the 'merge' option of 'config.get' is skipped when the salt command type is '"
-          ~ cli
-          ~ "'"
-        ) %}
-{%-   endif %}
-{%-   do _config.update(
-        {
-          "merge_opt": {},
-          "merge_msg": "",
-        }
-      ) %}
-{%- endif %}
+{#- `parse_matchers` returns a `list` of `dict` with #}
+{#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
+{#-  - option: specific to the type #}
+{#-  - query: which key is requested #}
+{#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
+{#-  - value: the result of the `salt[<query_method>](<query>)` #}
+{%- set map_matchers = parse_matchers(
+      matchers=map_sources,
+      config_get_strategy=_config["merge_strategy"],
+      log_prefix="map.jinja: "
+    )
+    | load_yaml %}
 
-{%- set query_map = {
-      "C": "config.get",
-      "G": "grains.get",
-      "I": "pillar.get",
-    } %}
+{%- for matcher in map_matchers %}
 
-{#- Process each `map.jinja` source #}
-{#- each source has a type: #}
-{#- - `Y` to load values from YAML files (the default when no type is set) #}
-{#- - `C` to lookup values with `config.get` #}
-{#- - `G` to lookup values with `grains.get` #}
-{#- - `I` to lookup values with `pillar.get` #}
-{#- The YAML type option can define query type to build the file name: #}
-{#- - `C` for query with `config.get` (the default when to query type is set) #}
-{#- - `G` for query with `grains.get` #}
-{#- - `I` for query with `pillar.get` #}
-{#- The `C`, `G` or `I` types can define the `SUB` option #}
-{#- to merge values in the sub key `mapdata.<key>` instead of directly in `mapdata` #}
-{%- for map_source in map_sources %}
-{%-   set source_parts = map_source.split('@') %}
-{%-   if source_parts|length == 1 %}
-{#-     By default we load YAML files for config looked up by `config.get` #}
-{%-     set source_type = "Y" %}
-{%-     set query_type = "C" %}
-{%-     set source_key = map_source %}
-{%-   elif source_parts[0][0] == "Y" %}
-{%-     set source_type = "Y" %}
-{%-     set query_type = source_parts[0].split(':')[1] | default("C") %}
-{%-     set source_key = source_parts[1] %}
-{%-   elif source_parts[0][0] in query_map.keys() %}
-{%-     set source_type = source_parts[0].split(':') | first %}
-{%-     set query_type = source_type %}
-{%-     set is_sub_key = source_parts[0].split(':')[1] | default(False) == "SUB" %}
-{%-     set source_key = source_parts[1] %}
-{%-   endif %}
-
-{%-   set query_method = query_map[query_type] %}
-
-{%-   if source_type in query_map.keys() %}
-{#-     Lookup source `<QUERY_METHOD>@key:to:query` #}
-{%-     if source_type == "C" %}
-{%-       set merge_opts = _config["merge_opt"] %}
-{%-       set merge_msg = _config["merge_msg"] %}
-{%-     else %}
-{#-       No merging strategy supported for `grains.get` and `pillar.get` #}
-{%-       set merge_opts = {} %}
-{%-       set merge_msg = "" %}
-{%-     endif %}
-
-{%-     do salt["log.debug"](
-          "map.jinja: retrieve '"
-          ~ source_key
-          ~ "' with '"
-          ~ query_method
-          ~ "'"
-          ~ merge_msg
-        ) %}
-{%-     set _config_get = salt[query_method](
-          source_key,
-          default={},
-          **merge_opts
-        ) %}
+{%-   if matcher.type in query_map.keys() %}
+{#-     Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
+{%-     set is_sub_key = matcher.option | default(False) == "SUB" %}
 
 {#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
 {%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
@@ -208,9 +139,9 @@
           "map.jinja: merge "
           ~ "sub key " * is_sub_key
           ~ "'"
-          ~ source_key
+          ~ matcher.query
           ~ "' retrieved with '"
-          ~ query_method
+          ~ matcher.query_method
           ~ "', merge: strategy='"
           ~ _strategy
           ~ "', lists='"
@@ -218,15 +149,17 @@
           ~ "'"
         ) %}
 
+{#-     empty value is an empty list, it must be an empty dict for `.update` #}
+{%-     set value = matcher.value | default({}, boolean=True) %}
 {%-     if is_sub_key %}
 {#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
-{%-       set _config_get = { source_key.rstrip(':lookup'): _config_get } %}
+{%-       set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
 {%-     endif %}
 {%-     do _config.update(
           {
             "stack": salt["slsutil.merge"](
               _config["stack"],
-              _config_get,
+              value,
               strategy=_strategy,
               merge_lists=_config["merge_lists"],
             )
@@ -235,20 +168,14 @@
 {%-   else %}
 {#-     Load YAML file matching the grain/pillar/... #}
 {#-     Fallback to use the source name as a direct filename #}
-{%-     do salt["log.debug"](
-          "map.jinja: lookup '"
-          ~ source_key
-          ~ "' with '"
-          ~ query_method
-          ~ "'"
-        ) %}
-{%-     set map_values = salt[query_method](source_key, []) %}
-
 {#-     Mangle `source_key` to use it as literal path #}
-{%-     if map_values | length == 0 %}
-{%-       set map_source_parts = source_key.split("/") %}
-{%-       set source_key = map_source_parts[0:-1] | join("/") %}
-{%-       set map_values = map_source_parts[-1].rstrip(".yaml") %}
+{%-     if matcher.value | length == 0 %}
+{%-       set query_parts = matcher.query.split("/") %}
+{%-       set map_dirname = query_parts[0:-1] | join("/") %}
+{%-       set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
+{%-     else %}
+{%-       set map_dirname = matcher.query %}
+{%-       set map_values = matcher.value %}
 {%-     endif %}
 
 {#-     Some configuration return list #}
@@ -256,10 +183,10 @@
 {%-       set map_values = [map_values] %}
 {%-     endif %}
 
-{#-     `source_key` can be an empty string with literal path like `myconf.yaml` #}
+{#-     `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
 {%-     set yaml_dir = [
           map_sources_dir,
-          source_key
+          map_dirname
         ]
         | select
         | join("/") %}

From d69556d5ae79a907d79351d4b9775e0ce2970b39 Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:47:11 +0100
Subject: [PATCH 4/8] feat(matchers): add delimiter option for source
 definitions

---
 docs/map.jinja.rst        | 47 ++++++++++++++++++++++++++++++++++++++-
 openssh/libmatchers.jinja | 32 ++++++++++++++++++--------
 openssh/map.jinja         |  1 +
 3 files changed, 70 insertions(+), 10 deletions(-)

diff --git a/docs/map.jinja.rst b/docs/map.jinja.rst
index dab217b..7c34b10 100644
--- a/docs/map.jinja.rst
+++ b/docs/map.jinja.rst
@@ -197,6 +197,47 @@ For the clients:
 
     Since we used ``Y:C@roles``, ``map.jinja`` will do a ``salt['config.get']('roles')`` to retrieve the roles so you could use any other method to bind roles to minions (`pillars`_ or `SDB`_) but `grains`_ seems to be the prefered method.
 
+Note for Microsoft Windows systems
+``````````````````````````````````
+
+If you have a minion running under windows, you can't use colon ``:`` as a delimiter for grain path query (see `bug 58726`_) in which case you should use an alternate delimiter:
+
+Modify ``/srv/salt/parameters/map_jinja.yaml`` to change the query for ``dns:domain`` to define the `alternate delimiter`_:
+
+.. code-block:: yaml
+
+    ---
+    values:
+      sources:
+        # default values
+        - "Y:G@osarch"
+        - "Y:G@os_family"
+        - "Y:G@os"
+        - "Y:G@osfinger"
+        - "C@{{ tplroot ~ ':lookup' }}"
+        - "C@{{ tplroot }}"
+
+        # Roles activate/deactivate things
+        # then thing are configured depending on environment
+        # So roles comes before `dns:domain`, `domain` and `id`
+        - "Y:C@roles"
+
+        # DNS domain configured (DHCP or resolv.conf)
+        - "Y:G:!@dns!domain"
+
+        # Based on minion ID
+        - "Y:G@domain"
+
+        # default values
+        - "Y:G@id"
+    ...
+
+And then, rename the directory:
+
+.. code-block:: console
+
+    mv /srv/salt/TEMPLATE/parameters/dns:domain/  '/srv/salt/TEMPLATE/parameters/dns!domain/'
+
 
 Format of configuration YAML files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -246,7 +287,7 @@ The ``map.jinja`` file uses several sources where to lookup parameter values. Th
 1. a global ``salt://parameters/map_jinja.yaml``
 2. a per formula ``salt://{{ tplroot }}/parameters/map_jinja.yaml``, it overrides the global configuration
 
-Each source definition has the form ``<TYPE>:<OPTION>@<KEY>`` where ``<TYPE>`` can be one of:
+Each source definition has the form ``[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<KEY>`` where ``<TYPE>`` can be one of:
 
 - ``Y`` to load values from YAML files from the `fileserver`_, this is the default when no type is defined
 - ``C`` to lookup values with `salt['config.get']`_
@@ -261,6 +302,8 @@ The YAML type option can define the query method to lookup the key value to buil
 
 The ``C``, ``G`` or ``I`` types can define the ``SUB`` option to store values in the sub key ``mapdata.<KEY>`` instead of directly in ``mapdata``.
 
+All types can define the ``<DELIMITER>`` option to use an `alternate delimiter`_ of the ``<KEY>``, for example: on windows system you can't use colon ``:`` for YAML file path name and you should use something else like exclamation mark ``!``.
+
 Finally, the ``<KEY>`` describes what to lookup to either build the YAML filename or gather values using one of the query methods.
 
 .. note::
@@ -432,6 +475,7 @@ This ``sls`` file expose a ``TEMPLATE`` context variable to the jinja template w
 .. _salt['config.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.config.html#salt.modules.config.get
 .. _salt['grains.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.grains.html#salt.modules.grains.get
 .. _salt['pillar.get']: https://docs.saltstack.com/en/latest/ref/modules/all/salt.modules.pillar.html#salt.modules.pillar.get
+.. _alternate delimiter: https://docs.saltstack.com/en/latest/topics/targeting/compound.html#alternate-delimiters
 .. _pillar.vault: https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.vault.html
 .. _pillars: https://docs.saltstack.com/en/latest/topics/pillar/
 .. _grains: https://docs.saltstack.com/en/latest/topics/grains/
@@ -445,3 +489,4 @@ This ``sls`` file expose a ``TEMPLATE`` context variable to the jinja template w
 .. _traverse: https://docs.saltstack.com/en/latest/topics/jinja/index.html#traverse
 .. _salt-ssh: https://docs.saltstack.com/en/latest/topics/ssh/
 .. _template-formula/TEMPLATE/config/file.sls: https://github.com/saltstack-formulas/template-formula/blob/master/TEMPLATE/config/file.sls
+.. _bug 58726: https://github.com/saltstack/salt/issues/58726
diff --git a/openssh/libmatchers.jinja b/openssh/libmatchers.jinja
index b5e768d..e9aaed3 100644
--- a/openssh/libmatchers.jinja
+++ b/openssh/libmatchers.jinja
@@ -14,9 +14,11 @@
 {#- When no part before `@` is provided: #}
 {#-   - define a filename path, noted `F` #}
 {#-   - use `salt["config.get"]`, noted `C` #}
+{#-   - use colon `:` delimiter for querying #}
 {%- set _defaults = {
       "type": "F",
       "query_type": "C",
+      "query_delimiter": ":"
     } %}
 
 {%- macro parse_matchers(
@@ -24,7 +26,7 @@
       config_get_strategy=None,
       log_prefix="libmatchers: "
     ) %}
-{#-   matcher format is `[<TYPE>[:<OPTION>]@]<KEY>` #}
+{#-   matcher format is `[<TYPE>[:<OPTION>[:DELIMITER]]@]<KEY>` #}
 {#-   each matcher has a type: #}
 {#-   - `F` to build a file name (the default when no type is set) #}
 {#-   - `C` to lookup values with `config.get` #}
@@ -34,6 +36,7 @@
 {#-   - `C` for query with `config.get` (the default when to query type is set) #}
 {#-   - `G` for query with `grains.get` #}
 {#-   - `I` for query with `pillar.get` #}
+{#-   With `DELIMITER`, you can choose a different delimiter when doing queries #}
 {%-   set parsed_matchers = [] %}
 {%-   for matcher in matchers %}
 {%-     do salt["log.debug"](
@@ -52,6 +55,7 @@
               "type": _defaults["type"],
               "option": None,
               "query_method": query_map[_defaults["query_type"]],
+              "query_delimiter": _defaults["query_delimiter"],
               "query": matcher
             }
           ) %}
@@ -80,6 +84,7 @@
               {
                 "type": metadatas[0],
                 "option": "C",
+                "query_delimiter": ":"
               }
             ) %}
 {%-         do salt["log.debug"](
@@ -94,6 +99,7 @@
               {
                 "type": metadatas[0],
                 "option": metadatas[1],
+                "query_delimiter": ":"
               }
             ) %}
 {%-         do salt["log.debug"](
@@ -108,6 +114,7 @@
               {
                 "type": metadatas[0],
                 "option": metadatas[1],
+                "query_delimiter": metadatas[2] | default(":", boolean=True)
               }
             ) %}
 {%-         do salt["log.debug"](
@@ -118,10 +125,12 @@
               | indent(4, True)
             ) %}
 {%-       elif metadatas | length == 4 %}
+{#-         The delimiter is `:` #}
 {%-         do parsed.update(
               {
                 "type": metadatas[0],
                 "option": metadatas[1],
+                "query_delimiter": ":"
               }
             ) %}
 {%-         do salt["log.debug"](
@@ -154,9 +163,14 @@
 
 {#-     Add `merge:` option to `salt["config.get"]` if configured #}
 {%-     if cli in ["minion", "local"] and parsed.query_method == "config.get" and config_get_strategy %}
-{%-       set merge_opt = {"merge": config_get_strategy} %}
-{%-       set merge_msg = (
-            ", merge: strategy='"
+{%-       set query_opts = {
+            "merge": config_get_strategy,
+            "delimiter": parsed.query_delimiter,
+          } %}
+{%-       set query_opts_msg = (
+            ", delimiter='"
+            ~ parsed.query_delimiter
+            ~ "', merge: strategy='"
             ~ config_get_strategy
             ~ "'"
           ) %}
@@ -164,13 +178,13 @@
 {%-       if cli not in ["minion", "local"] %}
 {%-         do salt["log.error"](
               log_prefix
-              ~ "the 'merge' option of 'config.get' is skipped when the salt command type is '"
+              ~ "the 'delimiter' and 'merge' options of 'config.get' are skipped when the salt command type is '"
               ~ cli
               ~ "'"
             ) %}
 {%-       endif %}
-{%-       set merge_opt = {} %}
-{%-       set merge_msg = "" %}
+{%-       set query_opts = {} %}
+{%-       set query_opts_msg = "" %}
 {%-     endif %}
 
 {%-     do salt["log.debug"](
@@ -180,12 +194,12 @@
           ~ "' with '"
           ~ parsed.query_method
           ~ "'"
-          ~ merge_msg
+          ~ query_opts_msg
         ) %}
 {%-     set values = salt[parsed.query_method](
           parsed.query,
           default=[],
-          **merge_opt
+          **query_opts
         ) %}
 {%-     do parsed.update(
           {
diff --git a/openssh/map.jinja b/openssh/map.jinja
index a12d3ea..3919d84 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -118,6 +118,7 @@
 {#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
 {#-  - option: specific to the type #}
 {#-  - query: which key is requested #}
+{#-  - query_delimiter: the separator between query component #}
 {#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
 {#-  - value: the result of the `salt[<query_method>](<query>)` #}
 {%- set map_matchers = parse_matchers(

From 568bb7ce4075ee376e8c49a45a1470d252f82ab9 Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:47:50 +0100
Subject: [PATCH 5/8] refactor(map): load `map.jinja` configuration with
 `libmapstack`

---
 openssh/libmapstack.jinja | 127 ++++++++++++++++++++++++++++++++++++++
 openssh/map.jinja         |  69 ++++-----------------
 2 files changed, 140 insertions(+), 56 deletions(-)
 create mode 100644 openssh/libmapstack.jinja

diff --git a/openssh/libmapstack.jinja b/openssh/libmapstack.jinja
new file mode 100644
index 0000000..9ea212b
--- /dev/null
+++ b/openssh/libmapstack.jinja
@@ -0,0 +1,127 @@
+{#- -*- coding: utf-8 -*- #}
+{#- vim: ft=jinja #}
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split("/")[0] %}
+
+{%- macro mapstack(
+      files,
+      defaults=None,
+      log_prefix="libmapstack: "
+    ) %}
+{#-
+    Load YAML files in the order of `files` and merge successively the
+    values with `defaults` if the file exists.
+
+    Parameters:
+      - `files`: list of files to load
+
+      - `defaults`: default values to start the merging, they are
+        considered built-ins
+
+      - `log_prefix`: prefix used in the log outputs, by default it is
+        `libmapstack: `
+
+    Each YAML file must conform to the following layout:
+      - a mandatory `values` key to store the configuration values
+
+      - two optional keys to configure the use of `salt.slsutil.merge`
+
+        - an optional `strategy` key to configure the merging strategy,
+          for example `strategy: 'recurse'`, the default is `smart`
+
+        - an optional `merge_lists` key to configure if lists should be
+          merged or overridden for the `recurse` and `overwrite`
+          strategies, for example `merge_lists: 'true'`
+#}
+{%-   set stack = defaults | default({"values": {} }, boolean=True) %}
+
+{%-   do salt["log.debug"](
+        log_prefix
+        ~ "built-in configuration:\n"
+        ~ {"values": defaults | traverse("values")}
+        | yaml(False)
+      ) %}
+
+{%-   for yaml_filename in files %}
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "load configuration values from "
+          ~ yaml_filename
+        ) %}
+
+{%-     load_yaml as yaml_values %}
+{%-       include yaml_filename ignore missing %}
+{%-     endload %}
+
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "loaded configuration values from "
+          ~ yaml_filename
+          ~ ":\n"
+          ~ yaml_values
+          | yaml(False)
+        ) %}
+
+{%-     if yaml_values %}
+{#-       Per YAML file `salt["slsutil.merge"]` options or use defaults #}
+{%-       set _strategy = yaml_values
+            | traverse(
+              "strategy",
+              defaults
+              | traverse(
+                "strategy",
+                "smart"
+              )
+            ) %}
+{%-       set _merge_lists = yaml_values
+            | traverse(
+              "merge_lists",
+              defaults
+              | traverse(
+                "merge_lists",
+                False
+              )
+            )
+            | to_bool %}
+
+{#-       Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
+{%-       do stack.update(
+            {
+              "values": salt["slsutil.merge"](
+                stack["values"],
+                yaml_values
+                | traverse("values", {}),
+                strategy=_strategy,
+                merge_lists=_merge_lists,
+              )
+            }
+          ) %}
+{%-       do salt["log.debug"](
+            log_prefix
+            ~ "merged configuration values from "
+            ~ yaml_filename
+            ~ ", merge: strategy='"
+            ~ _strategy
+            ~ "', merge_lists='"
+            ~ _merge_lists
+            ~ "':\n"
+            ~ {"values": stack["values"]}
+            | yaml(False)
+          ) %}
+{%-     endif %}
+
+{%-   endfor %}
+
+{%-   do salt["log.debug"](
+        log_prefix
+        ~ "final configuration values:\n"
+        ~ {"values": stack["values"]}
+        | yaml(False)
+      ) %}
+
+{#- Output stack as YAML, caller should use with something like #}
+{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
+{{ stack | yaml }}
+
+{%- endmacro %}
diff --git a/openssh/map.jinja b/openssh/map.jinja
index 3919d84..8280213 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -4,6 +4,7 @@
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
 {%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
+{%- from tplroot ~ "/libmapstack.jinja" import mapstack %}
 
 {#- Where to lookup parameters source files #}
 {%- set map_sources_dir = tplroot ~ "/parameters" %}
@@ -19,65 +20,22 @@
       "C@" ~ tplroot,
       "Y:G@id",
     ] %}
-{%- do salt["log.debug"](
-      "map.jinja: built-in configuration sources:\n"
-      ~ {"values": {
-          "map_jinja": {"sources": map_sources}
-          }
-        }
-      | yaml(False)
-    ) %}
 
 {#- Allow centralised map.jinja configuration #}
-{%- set _global_map_filename = "parameters/map_jinja.yaml" %}
-{%- do salt["log.debug"](
-      "map.jinja: load global map.jinja values from "
-      ~ _global_map_filename
-    ) %}
-{%- load_yaml as global_map_settings %}
-{%-   include _global_map_filename ignore missing %}
-{%- endload %}
-
-{%- if global_map_settings %}
-{%-   do salt["log.debug"](
-        "map.jinja: configure sources from global map.jinja configuration "
-        ~ _global_map_filename
-        ~ ":\n"
-        ~ {"map_jinja": global_map_settings}
-        | yaml(False)
-      ) %}
-{%-   set map_sources = global_map_settings
-        | traverse(
-          "values:sources",
-          map_sources,
-      ) %}
-{%- endif %}
-
+{%- set _global_param_filename = "parameters/map_jinja.yaml" %}
 {#- Allow per formula map.jinja configuration #}
-{%- set _map_filename = map_sources_dir ~ "/map_jinja.yaml" %}
-{%- do salt["log.debug"](
-      "map.jinja: load per formula map.jinja values from "
-      ~ _map_filename
-    ) %}
-{%- load_yaml as map_settings %}
-{%-   include _map_filename ignore missing %}
-{%- endload %}
+{%- set _formula_param_filename = map_sources_dir ~ "/map_jinja.yaml" %}
 
-{%- if map_settings %}
-{%-   do salt["log.debug"](
-        "map.jinja: configure sources from formula map.jinja configuration "
-        ~ _map_filename
-        ~ ":\n"
-        ~ {"map_jinja": map_settings}
-        | yaml(False)
-      ) %}
-{%-   set map_sources = map_settings
-      | traverse(
-        "values:sources",
-        map_sources,
-      ) %}
-{%- endif %}
+{%- set _map_settings = mapstack(
+      files=[_global_param_filename, _formula_param_filename],
+      defaults={
+        "values": {"sources": map_sources}
+      },
+      log_prefix="map.jinja configuration: ",
+    )
+    | load_yaml %}
 
+{%- set map_sources = _map_settings | traverse("values:sources") %}
 {%- do salt["log.debug"](
       "map.jinja: load parameters from sources:\n"
       ~ map_sources
@@ -101,8 +59,7 @@
 {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
 {%- do default_settings["values"].update(
       {
-        "map_jinja": map_settings
-        | traverse("values", {})
+        "map_jinja": _map_settings["values"]
       }
     ) %}
 

From 174bb68432366a449a8327a9dbb648271f123224 Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 17:57:30 +0100
Subject: [PATCH 6/8] refactor(map): load `defaults.jinja` configuration with
 `libmapstack`

---
 openssh/map.jinja | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/openssh/map.jinja b/openssh/map.jinja
index 8280213..367b24d 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -44,18 +44,12 @@
 
 {#- Load formula defaults values #}
 {%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
-{%- do salt["log.debug"](
-      "map.jinja: load per formula default values from "
-      ~ _defaults_filename
-    ) %}
-{%- load_yaml as default_settings %}
-{%-   include _defaults_filename ignore missing %}
-{%- endload %}
-
-{%- if not default_settings %}
-{%-   set default_settings = {'values': {} } %}
-{%- endif %}
-
+{%- set default_settings = mapstack(
+      files=[_defaults_filename],
+      defaults={"values": {} },
+      log_prefix="map.jinja defaults: ",
+    )
+    | load_yaml %}
 {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
 {%- do default_settings["values"].update(
       {

From ff6b56c4a4e282f41ddfc8f379f95096fea0553f Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 19:00:47 +0100
Subject: [PATCH 7/8] refactor(map): load formula configuration with
 `libmatchers`

---
 openssh/libmapstack.jinja | 319 +++++++++++++++++++++++++++++---------
 openssh/map.jinja         | 148 ++----------------
 2 files changed, 262 insertions(+), 205 deletions(-)

diff --git a/openssh/libmapstack.jinja b/openssh/libmapstack.jinja
index 9ea212b..bef5e5a 100644
--- a/openssh/libmapstack.jinja
+++ b/openssh/libmapstack.jinja
@@ -3,39 +3,104 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
+
+{%- set _default_config_dirs = [
+      "parameters/",
+      tplroot ~ "/parameters"
+    ] %}
 
 {%- macro mapstack(
-      files,
+      matchers,
       defaults=None,
+      dirs=_default_config_dirs,
       log_prefix="libmapstack: "
     ) %}
 {#-
-    Load YAML files in the order of `files` and merge successively the
-    values with `defaults` if the file exists.
+    Load configuration in the order of `matchers` and merge
+    successively the values with `defaults`.
+
+    The `matchers` are processed using `libmatchers.jinja` to select
+    the configuration sources from where the values are loaded.
 
     Parameters:
-      - `files`: list of files to load
 
-      - `defaults`: default values to start the merging, they are
-        considered built-ins
+      - `matchers`: list of matchers in the form
+        `[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`
+
+      - `defaults`: dictionary of default values to start the merging,
+        they are considered built-ins. It must conform to the same
+        layout as the YAML files: a mandatory `values` key and two
+        optional `strategy` and `merge_lists` keys.
+
+      - `dirs`: list of directory where to look-up the configuration
+        file matching the matchers, by default a global `salt://parameters/`
+        and a per formula `salt://<tplroot>/parameters`
 
       - `log_prefix`: prefix used in the log outputs, by default it is
         `libmapstack: `
 
-    Each YAML file must conform to the following layout:
+    Example: On a Debian system with `roles=["nginx/server", "telegraf"]`
+
+      {%- set settings = mapstack(
+            matchers=[
+              "Y:G@os_family",
+              "I@" ~ tplroot,
+              "Y:C@roles",
+            ],
+            dirs=["defaults", tplroot ~ "/parameters"],
+          )
+          | load_yaml %}
+
+      This will merge the values:
+
+        - starting with the default empty dictionary `{}` (no
+          `defaults` parameter)
+
+        - from the YAML files
+
+          - `salt://defaults/os_family/Debian.yaml`
+
+          - `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`
+
+        - from the pillar `salt["pillar.get"](tplroot)`
+
+        - from the `nginx/server` YAML files:
+
+          - `salt://defaults/roles/nginx/server.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`
+
+        - from the `telegraf` YAML files:
+
+          - `salt://defaults/roles/telegraf.yaml`
+
+          - `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`
+
+    Each YAML file and the `defaults` parameters must conform to the
+    following layout:
+
       - a mandatory `values` key to store the configuration values
 
       - two optional keys to configure the use of `salt.slsutil.merge`
 
-        - an optional `strategy` key to configure the merging strategy,
-          for example `strategy: 'recurse'`, the default is `smart`
+        - an optional `strategy` key to configure the merging
+          strategy, for example `strategy: 'recurse'`, the default is
+          `smart`
 
-        - an optional `merge_lists` key to configure if lists should be
-          merged or overridden for the `recurse` and `overwrite`
+        - an optional `merge_lists` key to configure if lists should
+          be merged or overridden for the `recurse` and `overwrite`
           strategies, for example `merge_lists: 'true'`
 #}
 {%-   set stack = defaults | default({"values": {} }, boolean=True) %}
 
+{#-   Build configuration file names based on matchers #}
+{%-   set matchers = parse_matchers(
+        matchers,
+        log_prefix=log_prefix
+      )
+      | load_yaml %}
+
 {%-   do salt["log.debug"](
         log_prefix
         ~ "built-in configuration:\n"
@@ -43,74 +108,182 @@
         | yaml(False)
       ) %}
 
-{%-   for yaml_filename in files %}
-{%-     do salt["log.debug"](
-          log_prefix
-          ~ "load configuration values from "
-          ~ yaml_filename
-        ) %}
-
-{%-     load_yaml as yaml_values %}
-{%-       include yaml_filename ignore missing %}
-{%-     endload %}
-
-{%-     do salt["log.debug"](
-          log_prefix
-          ~ "loaded configuration values from "
-          ~ yaml_filename
-          ~ ":\n"
-          ~ yaml_values
-          | yaml(False)
-        ) %}
-
-{%-     if yaml_values %}
-{#-       Per YAML file `salt["slsutil.merge"]` options or use defaults #}
-{%-       set _strategy = yaml_values
+{%-   for param_dir in dirs %}
+{%-     for matcher in matchers %}
+{#-       `slsutil.merge` options from #}
+{#-       1. the `value` #}
+{#-       2. the `defaults` #}
+{#-       3. the built-in #}
+{%-       set _strategy = matcher.value
+          | traverse(
+            "strategy",
+            defaults
             | traverse(
               "strategy",
-              defaults
-              | traverse(
-                "strategy",
-                "smart"
-              )
-            ) %}
-{%-       set _merge_lists = yaml_values
+              "smart"
+            )
+          ) %}
+{%-       set _merge_lists = matcher.value
+          | traverse(
+            "merge_lists",
+            defaults
             | traverse(
               "merge_lists",
-              defaults
-              | traverse(
-                "merge_lists",
-                False
-              )
+              False
             )
-            | to_bool %}
+          )
+          | to_bool %}
 
-{#-       Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
-{%-       do stack.update(
-            {
-              "values": salt["slsutil.merge"](
-                stack["values"],
-                yaml_values
-                | traverse("values", {}),
-                strategy=_strategy,
-                merge_lists=_merge_lists,
-              )
-            }
-          ) %}
-{%-       do salt["log.debug"](
-            log_prefix
-            ~ "merged configuration values from "
-            ~ yaml_filename
-            ~ ", merge: strategy='"
-            ~ _strategy
-            ~ "', merge_lists='"
-            ~ _merge_lists
-            ~ "':\n"
-            ~ {"values": stack["values"]}
-            | yaml(False)
-          ) %}
-{%-     endif %}
+{%-       if matcher.type in query_map.keys() %}
+{#-         No value is an empty list, must be a dict for `stack.update` #}
+{%-         set normalized_value = matcher.value | default({}, boolean=True) %}
 
+{#-         Merge in `mapdata.<query>` instead of directly in `mapdata` #}
+{%-         set is_sub_key = matcher.option | default(False) == "SUB" %}
+{%-         if is_sub_key %}
+{#-           Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
+{%-           set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
+{%-         else %}
+{%-           set value = normalized_value %}
+{%-         endif %}
+
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "merge "
+              ~ "sub key " * is_sub_key
+              ~ "'"
+              ~ matcher.query
+              ~ "' retrieved with '"
+              ~ matcher.query_method
+              ~ "', merge: strategy='"
+              ~ _strategy
+              ~ "', lists='"
+              ~ _merge_lists
+              ~ "':\n"
+              ~ value
+              | yaml(False)
+            ) %}
+
+{%-         do stack.update(
+              {
+                "values": salt["slsutil.merge"](
+                  stack["values"],
+                  value,
+                  strategy=_strategy,
+                  merge_lists=_merge_lists,
+                )
+              }
+            ) %}
+
+{%-       else %}
+{#-         Load YAML file matching the grain/pillar/... #}
+{#-         Fallback to use the source name as a direct filename #}
+
+{%-         if matcher.value | length == 0 %}
+{#-           Mangle `matcher.value` to use it as literal path #}
+{%-           set query_parts = matcher.query.split("/") %}
+{%-           set yaml_dirname = query_parts[0:-1] | join("/") %}
+{%-           set yaml_names = query_parts[-1] %}
+{%-         else %}
+{%-           set yaml_dirname = matcher.query %}
+{%-           set yaml_names = matcher.value %}
+{%-         endif %}
+
+{#-         Some configuration return list #}
+{%-         if yaml_names is string %}
+{%-           set yaml_names = [yaml_names] %}
+{%-         endif %}
+
+{#-         `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
+{%-         set yaml_dir = [
+              param_dir,
+              yaml_dirname
+            ]
+            | select
+            | join("/") %}
+
+{%-         for yaml_name in yaml_names %}
+{#-           Make sure to have a `.yaml` extension #}
+{#-           Use `.rpartition` to strip last `.yaml` in `dir.yaml/file.yaml` #}
+{%-           set yaml_filename = [
+                yaml_dir.rstrip("/"),
+                yaml_name.rpartition(".yaml")
+                | reject("equalto", ".yaml")
+                | join
+                ~ ".yaml"
+              ]
+              | select
+              | join("/") %}
+
+{%-           do salt["log.debug"](
+                log_prefix
+                ~ "load configuration values from "
+                ~ yaml_filename
+              ) %}
+{%-           load_yaml as yaml_values %}
+{%-             include yaml_filename ignore missing %}
+{%-           endload %}
+
+{%-           if yaml_values %}
+{%-             do salt["log.debug"](
+                  log_prefix
+                  ~ "loaded configuration values from "
+                  ~ yaml_name
+                  ~ ":\n"
+                  ~ yaml_values
+                  | yaml(False)
+                ) %}
+
+{#-             `slsutil.merge` options from #}
+{#-             1. the `value` #}
+{#-             2. the `defaults` #}
+{#-             3. the built-in #}
+{%-             set _strategy = yaml_values
+                  | traverse(
+                    "strategy",
+                    defaults
+                    | traverse(
+                      "strategy",
+                      "smart"
+                    )
+                  ) %}
+{%-             set _merge_lists = yaml_values
+                  | traverse(
+                    "merge_lists",
+                    defaults
+                    | traverse(
+                      "merge_lists",
+                      False
+                    )
+                  )
+                  | to_bool %}
+{%-             do stack.update(
+                  {
+                    "values": salt["slsutil.merge"](
+                      stack["values"],
+                      yaml_values
+                      | traverse("values", {}),
+                      strategy=_strategy,
+                      merge_lists=_merge_lists,
+                    )
+                  }
+                ) %}
+{%-             do salt["log.debug"](
+                  log_prefix
+                  ~ "merged configuration values from "
+                  ~ yaml_name
+                  ~ ", merge: strategy='"
+                  ~ _strategy
+                  ~ "', merge_lists='"
+                  ~ _merge_lists
+                  ~ "':\n"
+                  ~ {"values": stack["values"]}
+                  | yaml(False)
+                ) %}
+{%-           endif %}
+{%-         endfor %}
+{%-       endif %}
+{%-     endfor %}
 {%-   endfor %}
 
 {%-   do salt["log.debug"](
@@ -121,7 +294,7 @@
       ) %}
 
 {#- Output stack as YAML, caller should use with something like #}
-{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
+{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
 {{ stack | yaml }}
 
 {%- endmacro %}
diff --git a/openssh/map.jinja b/openssh/map.jinja
index 367b24d..9fdba2b 100644
--- a/openssh/map.jinja
+++ b/openssh/map.jinja
@@ -3,11 +3,10 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
-{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
 {%- from tplroot ~ "/libmapstack.jinja" import mapstack %}
 
 {#- Where to lookup parameters source files #}
-{%- set map_sources_dir = tplroot ~ "/parameters" %}
+{%- set formula_param_dir = tplroot ~ "/parameters" %}
 
 {#- List of sources to lookup for parameters #}
 {#- Fallback to previously used grains plus minion `id` #}
@@ -21,13 +20,8 @@
       "Y:G@id",
     ] %}
 
-{#- Allow centralised map.jinja configuration #}
-{%- set _global_param_filename = "parameters/map_jinja.yaml" %}
-{#- Allow per formula map.jinja configuration #}
-{%- set _formula_param_filename = map_sources_dir ~ "/map_jinja.yaml" %}
-
 {%- set _map_settings = mapstack(
-      files=[_global_param_filename, _formula_param_filename],
+      matchers=["map_jinja.yaml"],
       defaults={
         "values": {"sources": map_sources}
       },
@@ -42,136 +36,26 @@
       | yaml(False)
     ) %}
 
-{#- Load formula defaults values #}
-{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %}
-{%- set default_settings = mapstack(
-      files=[_defaults_filename],
-      defaults={"values": {} },
-      log_prefix="map.jinja defaults: ",
+{#- Load formula parameters values #}
+{%- set _formula_matchers = ["defaults.yaml"] + map_sources %}
+{%- set _formula_settings = mapstack(
+      matchers=_formula_matchers,
+      dirs=[formula_param_dir],
+      defaults={
+        "values": {},
+        "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
+        "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
+      },
+      log_prefix="map.jinja: ",
     )
     | load_yaml %}
+
 {#- Make sure to track `map.jinja` configuration with `_mapdata` #}
-{%- do default_settings["values"].update(
+{%- do _formula_settings["values"].update(
       {
         "map_jinja": _map_settings["values"]
       }
     ) %}
 
-{#- Work around assignment inside for loop #}
-{#- load configuration values used in `config.get` merging strategies #}
-{%- set _config = {
-      "stack": default_settings.get("values", {}),
-      "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
-      "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
-    } %}
-
-{#- `parse_matchers` returns a `list` of `dict` with #}
-{#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
-{#-  - option: specific to the type #}
-{#-  - query: which key is requested #}
-{#-  - query_delimiter: the separator between query component #}
-{#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
-{#-  - value: the result of the `salt[<query_method>](<query>)` #}
-{%- set map_matchers = parse_matchers(
-      matchers=map_sources,
-      config_get_strategy=_config["merge_strategy"],
-      log_prefix="map.jinja: "
-    )
-    | load_yaml %}
-
-{%- for matcher in map_matchers %}
-
-{%-   if matcher.type in query_map.keys() %}
-{#-     Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
-{%-     set is_sub_key = matcher.option | default(False) == "SUB" %}
-
-{#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
-{%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
-{%-     do salt["log.debug"](
-          "map.jinja: merge "
-          ~ "sub key " * is_sub_key
-          ~ "'"
-          ~ matcher.query
-          ~ "' retrieved with '"
-          ~ matcher.query_method
-          ~ "', merge: strategy='"
-          ~ _strategy
-          ~ "', lists='"
-          ~ _config["merge_lists"]
-          ~ "'"
-        ) %}
-
-{#-     empty value is an empty list, it must be an empty dict for `.update` #}
-{%-     set value = matcher.value | default({}, boolean=True) %}
-{%-     if is_sub_key %}
-{#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
-{%-       set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
-{%-     endif %}
-{%-     do _config.update(
-          {
-            "stack": salt["slsutil.merge"](
-              _config["stack"],
-              value,
-              strategy=_strategy,
-              merge_lists=_config["merge_lists"],
-            )
-          }
-        ) %}
-{%-   else %}
-{#-     Load YAML file matching the grain/pillar/... #}
-{#-     Fallback to use the source name as a direct filename #}
-{#-     Mangle `source_key` to use it as literal path #}
-{%-     if matcher.value | length == 0 %}
-{%-       set query_parts = matcher.query.split("/") %}
-{%-       set map_dirname = query_parts[0:-1] | join("/") %}
-{%-       set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
-{%-     else %}
-{%-       set map_dirname = matcher.query %}
-{%-       set map_values = matcher.value %}
-{%-     endif %}
-
-{#-     Some configuration return list #}
-{%-     if map_values is string %}
-{%-       set map_values = [map_values] %}
-{%-     endif %}
-
-{#-     `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
-{%-     set yaml_dir = [
-          map_sources_dir,
-          map_dirname
-        ]
-        | select
-        | join("/") %}
-
-{%-     for map_value in map_values %}
-{%-       set yamlfile = [
-            yaml_dir,
-            map_value ~ ".yaml"
-          ]
-          | join("/") %}
-{%-       do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
-{%-       load_yaml as loaded_values %}
-{%-         include yamlfile ignore missing %}
-{%-       endload %}
-
-{%-       if loaded_values %}
-{#-         Merge loaded values on the stack #}
-{%-         do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
-{%-         do _config.update(
-              {
-                "stack": salt["slsutil.merge"](
-                  _config["stack"],
-                  loaded_values.get("values", {}),
-                  strategy=loaded_values.get("strategy", "smart"),
-                  merge_lists=loaded_values.get("merge_lists", False)
-                  | to_bool,
-                )
-              }
-            ) %}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
-{%- endfor %}
-
 {%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
-{%- set mapdata = _config["stack"] %}
+{%- set mapdata = _formula_settings["values"] %}

From 7de2d6fd756b3e4b7154e660b639d7ce6edb8cfe Mon Sep 17 00:00:00 2001
From: Daniel Dehennin <daniel.dehennin@baby-gnu.org>
Date: Mon, 11 Jan 2021 19:01:08 +0100
Subject: [PATCH 8/8] style(mapstack): variables in macro can't be exported

No need to prefix some of them with underscore `_`.
---
 openssh/libmapstack.jinja | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/openssh/libmapstack.jinja b/openssh/libmapstack.jinja
index bef5e5a..7f9ab38 100644
--- a/openssh/libmapstack.jinja
+++ b/openssh/libmapstack.jinja
@@ -114,7 +114,7 @@
 {#-       1. the `value` #}
 {#-       2. the `defaults` #}
 {#-       3. the built-in #}
-{%-       set _strategy = matcher.value
+{%-       set strategy = matcher.value
           | traverse(
             "strategy",
             defaults
@@ -123,7 +123,7 @@
               "smart"
             )
           ) %}
-{%-       set _merge_lists = matcher.value
+{%-       set merge_lists = matcher.value
           | traverse(
             "merge_lists",
             defaults
@@ -156,9 +156,9 @@
               ~ "' retrieved with '"
               ~ matcher.query_method
               ~ "', merge: strategy='"
-              ~ _strategy
+              ~ strategy
               ~ "', lists='"
-              ~ _merge_lists
+              ~ merge_lists
               ~ "':\n"
               ~ value
               | yaml(False)
@@ -169,8 +169,8 @@
                 "values": salt["slsutil.merge"](
                   stack["values"],
                   value,
-                  strategy=_strategy,
-                  merge_lists=_merge_lists,
+                  strategy=strategy,
+                  merge_lists=merge_lists,
                 )
               }
             ) %}
@@ -238,7 +238,7 @@
 {#-             1. the `value` #}
 {#-             2. the `defaults` #}
 {#-             3. the built-in #}
-{%-             set _strategy = yaml_values
+{%-             set strategy = yaml_values
                   | traverse(
                     "strategy",
                     defaults
@@ -247,7 +247,7 @@
                       "smart"
                     )
                   ) %}
-{%-             set _merge_lists = yaml_values
+{%-             set merge_lists = yaml_values
                   | traverse(
                     "merge_lists",
                     defaults
@@ -263,8 +263,8 @@
                       stack["values"],
                       yaml_values
                       | traverse("values", {}),
-                      strategy=_strategy,
-                      merge_lists=_merge_lists,
+                      strategy=strategy,
+                      merge_lists=merge_lists,
                     )
                   }
                 ) %}
@@ -273,9 +273,9 @@
                   ~ "merged configuration values from "
                   ~ yaml_name
                   ~ ", merge: strategy='"
-                  ~ _strategy
+                  ~ strategy
                   ~ "', merge_lists='"
-                  ~ _merge_lists
+                  ~ merge_lists
                   ~ "':\n"
                   ~ {"values": stack["values"]}
                   | yaml(False)