feat(map): update to v4 “map.jinja”

The `map.jinja` now exports a single variable called `mapdata`.

We extract the `openssh`, `sshd_config` and `ssh_config` from it to
minimize the changes to `.sls` files.

openssh/_mapdata/init.sls

@@ -3,18 +3,11 @@
 ---
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- from tplroot ~ "/map.jinja" import openssh with context %}
-{%- from tplroot ~ "/map.jinja" import ssh_config with context %}
-{%- from tplroot ~ "/map.jinja" import sshd_config with context %}
+{%- from tplroot ~ "/map.jinja" import mapdata with context %}
 
 {%- set output_file = '/tmp/salt_mapdata_dump.yaml' %}
-{%- set map = {
-      'openssh': openssh,
-      'ssh_config': ssh_config,
-      'sshd_config': sshd_config,
-    } %}
 
-{%- do salt['log.debug']( map | yaml(False) ) %}
+{%- do salt['log.debug']( mapdata | yaml(False) ) %}
 
 {{ tplroot }}-mapdata-dump:
   file.managed:
@@ -22,4 +15,4 @@
     - source: salt://{{ tplroot }}/_mapdata/_mapdata.jinja
     - template: jinja
     - context:
-        map: {{ map | yaml }}
+        map: {{ mapdata | yaml }}

openssh/auth.sls

@@ -29,7 +29,10 @@
 
 include:
   - openssh
-{%- from "openssh/map.jinja" import openssh, sshd_config with context -%}
+{%- from "openssh/map.jinja" import mapdata with context -%}
+{%- set openssh = mapdata.openssh %}
+{%- set sshd_config = mapdata.sshd_config %}
+
 {%- set auth = openssh.get('auth', {}) -%}
 {%- for identifier,keys in auth.items() -%}
   {%- for key in keys -%}

openssh/auth_map.sls

@@ -1,7 +1,9 @@
 include:
   - openssh
 
-{%  from "openssh/map.jinja" import openssh, sshd_config with context -%}
+{%  from "openssh/map.jinja" import mapdata with context -%}
+{%- set openssh = mapdata.openssh %}
+{%- set sshd_config = mapdata.sshd_config %}
 {%- set authorized_keys_file = sshd_config.get("AuthorizedKeysFile", None) %}
 
 {%- for store, config in openssh.get("auth_map", {}).items() %}

openssh/banner.sls

@@ -1,6 +1,7 @@
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- from tplroot ~ "/map.jinja" import openssh with context %}
+{%- from tplroot ~ "/map.jinja" import mapdata with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch %}
+{%- set openssh = mapdata.openssh %}
 
 include:
   - openssh

openssh/client.sls

@@ -1,4 +1,5 @@
-{% from "openssh/map.jinja" import openssh with context %}
+{% from "openssh/map.jinja" import mapdata with context %}
+{%- set openssh = mapdata.openssh %}
 
 openssh_client:
   pkg.installed:

openssh/config.sls

@@ -1,6 +1,9 @@
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- from tplroot ~ "/map.jinja" import openssh, ssh_config, sshd_config with context %}
+{%- from tplroot ~ "/map.jinja" import mapdata with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch %}
+{%- set openssh = mapdata.openssh %}
+{%- set sshd_config = mapdata.sshd_config %}
+{%- set ssh_config = mapdata.ssh_config %}
 
 
 include:

openssh/config_ini.sls

@@ -1,4 +1,6 @@
-{%- from "openssh/map.jinja" import openssh, sshd_config with context %}
+{%- from "openssh/map.jinja" import mapdata with context %}
+{%- set openssh = mapdata.openssh %}
+{%- set sshd_config = mapdata.sshd_config %}
 
 include:
   - openssh

openssh/init.sls

@@ -1,4 +1,5 @@
-{% from "openssh/map.jinja" import openssh with context %}
+{% from "openssh/map.jinja" import mapdata with context %}
+{%- set openssh = mapdata.openssh %}
 
 openssh:
   {% if openssh.server is defined %}

openssh/known_hosts.sls

@@ -1,6 +1,7 @@
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- from tplroot ~ "/map.jinja" import openssh with context %}
+{%- from tplroot ~ "/map.jinja" import mapdata with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch %}
+{%- set openssh = mapdata.openssh %}
 
 ensure dig is available:
   pkg.installed:

openssh/libsaltcli.jinja

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# vim: ft=jinja
+
+{#- Get the relevant values from the `opts` dict #}
+{%- set opts_cli = opts.get('__cli', '') %}
+{%- set opts_masteropts_cli = opts | traverse('__master_opts__:__cli', '') %}
+
+{#- Determine the type of salt command being run #}
+{%- if opts_cli == 'salt-minion' %}
+{%-   set cli = 'minion' %}
+{%- elif opts_cli == 'salt-call' %}
+{%-   set cli = 'ssh' if opts_masteropts_cli in ('salt-ssh', 'salt-master') else 'local' %}
+{%- else %}
+{%-   set cli = 'unknown' %}
+{%- endif %}
+{%- do salt['log.debug']('[libsaltcli] the salt command type has been identified to be: ' ~ cli) %}

openssh/map.jinja

@@ -1,22 +1,176 @@
 # -*- coding: utf-8 -*-
 # vim: ft=jinja
 
-{#- Start imports as #}
-{% import_yaml 'openssh/defaults.yaml' as default_settings %}
-{% import_yaml 'openssh/osfamilymap.yaml' as osfamilymap %}
-{% import_yaml 'openssh/osmap.yaml' as osmap %}
-{% import_yaml 'openssh/osfingermap.yaml' as osfingermap %}
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libsaltcli.jinja" import cli with context %}
 
-{% set defaults = salt['grains.filter_by'](default_settings,
-    default='default',
-    merge=salt['grains.filter_by'](osfamilymap, grain='os_family',
-      merge=salt['grains.filter_by'](osmap, grain='os',
-        merge=salt['grains.filter_by'](osfingermap, grain='osfinger')
-      )
-    )
+{#- Where to lookup parameters source files #}
+{%- set map_sources_dir = tplroot | path_join("parameters") %}
+
+{#- Load defaults first to allow per formula default map.jinja configuration #}
+{%- set _defaults_filename = map_sources_dir | path_join("defaults.yaml") %}
+{%- do salt["log.debug"](
+      "map.jinja: initialise parameters from "
+      ~ _defaults_filename
+    ) %}
+{%- import_yaml _defaults_filename as default_settings %}
+
+{#- List of sources to lookup for parameters #}
+{%- do salt["log.debug"]("map.jinja: lookup 'map_jinja' configuration sources") %}
+{#- Fallback to previously used grains plus minion `id` #}
+{%- set map_sources = [
+      "osarch",
+      "os_family",
+      "os",
+      "osfinger",
+      "config_get_lookup",
+      "config_get",
+      "id",
+    ] %}
+{#- Configure map.jinja from defaults.yaml #}
+{%- set map_sources = default_settings | traverse(
+      "values:map_jinja:sources",
+      map_sources,
     ) %}
 
-{#- merge the openssh pillar #}
-{% set openssh = salt['pillar.get']('openssh', default=defaults['openssh'], merge=True) %}
-{% set ssh_config = salt['pillar.get']('ssh_config', default=defaults['ssh_config'], merge=True) %}
-{% set sshd_config = salt['pillar.get']('sshd_config', default=defaults['sshd_config'], merge=True) %}
+{#- Lookup global sources #}
+{%- set map_sources = salt["config.get"]("map_jinja:sources", map_sources) %}
+{#- Lookup per formula sources #}
+{%- set map_sources = salt["config.get"](
+      tplroot ~ ":map_jinja:sources",
+      map_sources,
+    ) %}
+
+{%- do salt["log.debug"](
+      "map.jinja: load parameters with sources from "
+      ~ map_sources
+    ) %}
+
+
+{#- Work around assignment inside for loop #}
+{#- load configuration values used in `config.get` merging strategies #}
+{%- set _config = {
+      "stack": default_settings.get("values", {}),
+      "merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
+      "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
+    } %}
+
+{#- the `config.get` merge option only works for `minion` or `local` salt command types #}
+{%- if cli in ["minion", "local"] %}
+{%-   do _config.update(
+        {
+          "merge_opt": {"merge": _config["merge_strategy"]},
+          "merge_msg": ", merge: strategy='" ~ _config["merge_strategy"] ~ "'",
+        }
+      ) %}
+{#- the `config.get` merge option is not available for `ssh` or `unknown` salt command types #}
+{%- else %}
+{%-   if _config["merge_strategy"] %}
+{%-     do salt["log.error"](
+          "map.jinja: the 'merge' option of 'config.get' is skipped when the salt command type is '"
+          ~ cli
+          ~ "'"
+        ) %}
+{%-   endif %}
+{%-   do _config.update(
+        {
+          "merge_opt": {},
+          "merge_msg": "",
+        }
+      ) %}
+{%- endif %}
+
+
+{#- process each `map.jinja` source #}
+{%- for map_source in map_sources %}
+{%-   if map_source  in ["config_get", "config_get_lookup"] %}
+{%-     set _config_key = {
+          "config_get": tplroot,
+          "config_get_lookup": tplroot ~ ":lookup",
+        }.get(map_source) %}
+{%-     set _config_type = {
+          "config_get": "configuration",
+          "config_get_lookup": "lookup",
+        }.get(map_source) %}
+
+{%-     do salt["log.debug"](
+          "map.jinja: retrieve formula "
+          ~ _config_type
+          ~ " with 'config.get'"
+          ~ _config["merge_msg"]
+        ) %}
+{%-     set _config_get = salt["config.get"](
+          _config_key, default={}, **_config["merge_opt"]
+        ) %}
+
+{#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
+{%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
+{%-     do salt["log.debug"](
+          "map.jinja: merge formula "
+          ~ _config_type
+          ~ " retrieved with 'config.get'"
+          ~ ", merge: strategy='"
+          ~ _strategy
+          ~ "', lists='"
+          ~ _config["merge_lists"]
+          ~ "'"
+        ) %}
+{%-     do _config.update(
+          {
+            "stack": salt["slsutil.merge"](
+              _config["stack"],
+              _config_get,
+              strategy=_strategy,
+              merge_lists=_config["merge_lists"],
+            )
+          }
+        ) %}
+{%-   else %}
+{#-     Lookup the grain/pillar/... #}
+{#-     Fallback to use the source name as a direct filename #}
+{%-     set map_values = salt["config.get"](map_source, []) %}
+
+{#-     Mangle `map_source` to use it as literal path #}
+{%-     if map_values | length == 0 %}
+{%-       set map_source_parts = map_source.split("/") %}
+{%-       set map_source = map_source_parts[0:-1] | join("/") %}
+{%-       set map_values = map_source_parts[-1].rstrip(".yaml") %}
+{%-     endif %}
+
+{#-     Some configuration return list #}
+{%-     if map_values is string %}
+{%-       set map_values = [map_values] %}
+{%-     endif %}
+
+{%-     for map_value in map_values %}
+{%-       set yamlfile = map_sources_dir | path_join(
+            map_source,
+            map_value ~ ".yaml",
+          ) %}
+{%-       do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
+{%-       load_yaml as loaded_values %}
+{%-         include yamlfile ignore missing %}
+{%-       endload %}
+
+{%-       if loaded_values %}
+{#-         Merge loaded values on the stack #}
+{%-         do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
+{%-         do _config.update(
+              {
+                "stack": salt["slsutil.merge"](
+                  _config["stack"],
+                  loaded_values.get("values", {}),
+                  strategy=loaded_values.get("strategy", "smart"),
+                  merge_lists=loaded_values.get("merge_lists", False)
+                  | to_bool,
+                )
+              }
+            ) %}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
+{%- endfor %}
+
+{%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
+{%- set mapdata = _config["stack"] %}

openssh/moduli.sls

@@ -1,4 +1,5 @@
-{% from "openssh/map.jinja" import openssh with context %}
+{% from "openssh/map.jinja" import mapdata with context %}
+{%- set openssh = mapdata.openssh %}
 
 {% set moduli = salt['pillar.get']('openssh:moduli', False) -%}
 {% set moduli_source = salt['pillar.get']('openssh:moduli_source', False) -%}

openssh/osfamilymap.yaml

@@ -1,71 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: ft=yaml
----
-Arch:
-  openssh:
-    server: openssh
-    client: openssh
-    service: sshd
-    dig_pkg: bind
-  sshd_config:
-    Subsystem: sftp /usr/lib/ssh/sftp-server
-
-Debian:
-  openssh:
-    server: openssh-server
-    client: openssh-client
-    service: ssh
-  sshd_config:
-    Subsystem: sftp /usr/lib/openssh/sftp-server
-
-FreeBSD:
-  openssh:
-    service: sshd
-    dig_pkg: bind-tools
-    sshd_config_group: wheel
-    ssh_config_group: wheel
-  sshd_config:
-    Subsystem: sftp /usr/libexec/sftp-server
-
-Gentoo:
-  openssh:
-    server: net-misc/openssh
-    client: net-misc/openssh
-    service: sshd
-    dig_pkg: net-dns/bind-tools
-
-OpenBSD:
-  openssh:
-    service: sshd
-    sshd_config_group: wheel
-    ssh_config_group: wheel
-  sshd_config:
-    Subsystem: sftp /usr/libexec/sftp-server
-
-RedHat:
-  openssh:
-    server: openssh-server
-    client: openssh-clients
-    service: sshd
-    dig_pkg: bind-utils
-  sshd_config:
-    Subsystem: sftp /usr/libexec/openssh/sftp-server
-
-Solaris:
-  openssh:
-    service: network/ssh
-    sshd_config_group: root
-    ssh_config_group: root
-    dig_pkg: bind
-    sshd_binary: /usr/lib/ssh/sshd
-  sshd_config:
-    Subsystem: sftp internal-sftp
-
-Suse:
-  openssh:
-    server: openssh
-    client: openssh
-    service: sshd
-    dig_pkg: bind-utils
-  sshd_config:
-    Subsystem: sftp /usr/lib/ssh/sftp-server

openssh/osfingermap.yaml

@@ -1,7 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: ft=yaml
----
-Ubuntu-18.04: {}
-CentOS-6:
-  openssh:
-    host_key_algos: ecdsa,rsa

openssh/osmap.yaml

@@ -1,4 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: ft=yaml
----
-FreeBSD: {}

openssh/parameters/defaults.yaml

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
-default:
+values:
   openssh:
     sshd_enable: true
     sshd_binary: /usr/sbin/sshd
@@ -31,3 +31,4 @@ default:
 
   sshd_config: {}
   ssh_config: {}
+...

openssh/parameters/os_family/Arch.yaml

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Arch.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    server: openssh
+    client: openssh
+    service: sshd
+    dig_pkg: bind
+  sshd_config:
+    Subsystem: sftp /usr/lib/ssh/sftp-server
+...

openssh/parameters/os_family/Debian.yaml

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Debian.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    server: openssh-server
+    client: openssh-client
+    service: ssh
+  sshd_config:
+    Subsystem: sftp /usr/lib/openssh/sftp-server
+...

openssh/parameters/os_family/FreeBSD.yaml

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == FreeBSD.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    service: sshd
+    dig_pkg: bind-tools
+    sshd_config_group: wheel
+    ssh_config_group: wheel
+  sshd_config:
+    Subsystem: sftp /usr/libexec/sftp-server
+...

openssh/parameters/os_family/Gentoo.yaml

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Gentoo.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    server: net-misc/openssh
+    client: net-misc/openssh
+    service: sshd
+    dig_pkg: net-dns/bind-tools
+...

openssh/parameters/os_family/OpenBSD.yaml

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == OpenBSD.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    service: sshd
+    sshd_config_group: wheel
+    ssh_config_group: wheel
+  sshd_config:
+    Subsystem: sftp /usr/libexec/sftp-server
+...

openssh/parameters/os_family/RedHat.yaml

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == RedHat.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    server: openssh-server
+    client: openssh-clients
+    service: sshd
+    dig_pkg: bind-utils
+  sshd_config:
+    Subsystem: sftp /usr/libexec/openssh/sftp-server
+...

openssh/parameters/os_family/Solaris.yaml

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Solaris.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    service: network/ssh
+    sshd_config_group: root
+    ssh_config_group: root
+    dig_pkg: bind
+    sshd_binary: /usr/lib/ssh/sshd
+  sshd_config:
+    Subsystem: sftp internal-sftp
+...

openssh/parameters/os_family/Suse.yaml

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Suse.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    server: openssh
+    client: openssh
+    service: sshd
+    dig_pkg: bind-utils
+  sshd_config:
+    Subsystem: sftp /usr/lib/ssh/sftp-server
+...

openssh/parameters/osfinger/CentOS-6.yaml

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('osfinger') == CentOS-6.
+# You just need to add the key:values for this `osfinger` that differ
+# from `defaults.yaml`.
+#
+# If you do not need to provide defaults via the `osfinger` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  openssh:
+    host_key_algos: ecdsa,rsa
+...