Merge pull request #186 from baby-gnu/feature/v4-map.jinja

feat(map): update to v4 and add config.get lookup from multiple roots
This commit is contained in:
Imran Iqbal 2020-08-01 14:51:21 +01:00 committed by GitHub
commit db67ce6f42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 492 additions and 119 deletions

View File

@ -3,18 +3,11 @@
---
{#- Get the `tplroot` from `tpldir` #}
{%- set tplroot = tpldir.split('/')[0] %}
{%- from tplroot ~ "/map.jinja" import openssh with context %}
{%- from tplroot ~ "/map.jinja" import ssh_config with context %}
{%- from tplroot ~ "/map.jinja" import sshd_config with context %}
{%- from tplroot ~ "/map.jinja" import mapdata with context %}
{%- set output_file = '/tmp/salt_mapdata_dump.yaml' %}
{%- set map = {
'openssh': openssh,
'ssh_config': ssh_config,
'sshd_config': sshd_config,
} %}
{%- do salt['log.debug']( map | yaml(False) ) %}
{%- do salt['log.debug']( mapdata | yaml(False) ) %}
{{ tplroot }}-mapdata-dump:
file.managed:
@ -22,4 +15,4 @@
- source: salt://{{ tplroot }}/_mapdata/_mapdata.jinja
- template: jinja
- context:
map: {{ map | yaml }}
map: {{ mapdata | yaml }}

View File

@ -29,7 +29,10 @@
include:
- openssh
{%- from "openssh/map.jinja" import openssh, sshd_config with context -%}
{%- from "openssh/map.jinja" import mapdata with context -%}
{%- set openssh = mapdata.openssh %}
{%- set sshd_config = mapdata.sshd_config %}
{%- set auth = openssh.get('auth', {}) -%}
{%- for identifier,keys in auth.items() -%}
{%- for key in keys -%}

View File

@ -1,7 +1,9 @@
include:
- openssh
{% from "openssh/map.jinja" import openssh, sshd_config with context -%}
{% from "openssh/map.jinja" import mapdata with context -%}
{%- set openssh = mapdata.openssh %}
{%- set sshd_config = mapdata.sshd_config %}
{%- set authorized_keys_file = sshd_config.get("AuthorizedKeysFile", None) %}
{%- for store, config in openssh.get("auth_map", {}).items() %}

View File

@ -1,6 +1,7 @@
{%- set tplroot = tpldir.split('/')[0] %}
{%- from tplroot ~ "/map.jinja" import openssh with context %}
{%- from tplroot ~ "/map.jinja" import mapdata with context %}
{%- from tplroot ~ "/libtofs.jinja" import files_switch %}
{%- set openssh = mapdata.openssh %}
include:
- openssh

View File

@ -1,4 +1,5 @@
{% from "openssh/map.jinja" import openssh with context %}
{% from "openssh/map.jinja" import mapdata with context %}
{%- set openssh = mapdata.openssh %}
openssh_client:
pkg.installed:

View File

@ -1,6 +1,9 @@
{%- set tplroot = tpldir.split('/')[0] %}
{%- from tplroot ~ "/map.jinja" import openssh, ssh_config, sshd_config with context %}
{%- from tplroot ~ "/map.jinja" import mapdata with context %}
{%- from tplroot ~ "/libtofs.jinja" import files_switch %}
{%- set openssh = mapdata.openssh %}
{%- set sshd_config = mapdata.sshd_config %}
{%- set ssh_config = mapdata.ssh_config %}
include:

View File

@ -1,4 +1,6 @@
{%- from "openssh/map.jinja" import openssh, sshd_config with context %}
{%- from "openssh/map.jinja" import mapdata with context %}
{%- set openssh = mapdata.openssh %}
{%- set sshd_config = mapdata.sshd_config %}
include:
- openssh

View File

@ -1,4 +1,5 @@
{% from "openssh/map.jinja" import openssh with context %}
{% from "openssh/map.jinja" import mapdata with context %}
{%- set openssh = mapdata.openssh %}
openssh:
{% if openssh.server is defined %}

View File

@ -1,6 +1,7 @@
{%- set tplroot = tpldir.split('/')[0] %}
{%- from tplroot ~ "/map.jinja" import openssh with context %}
{%- from tplroot ~ "/map.jinja" import mapdata with context %}
{%- from tplroot ~ "/libtofs.jinja" import files_switch %}
{%- set openssh = mapdata.openssh %}
ensure dig is available:
pkg.installed:

16
openssh/libsaltcli.jinja Normal file
View File

@ -0,0 +1,16 @@
# -*- coding: utf-8 -*-
# vim: ft=jinja
{#- Get the relevant values from the `opts` dict #}
{%- set opts_cli = opts.get('__cli', '') %}
{%- set opts_masteropts_cli = opts | traverse('__master_opts__:__cli', '') %}
{#- Determine the type of salt command being run #}
{%- if opts_cli == 'salt-minion' %}
{%- set cli = 'minion' %}
{%- elif opts_cli == 'salt-call' %}
{%- set cli = 'ssh' if opts_masteropts_cli in ('salt-ssh', 'salt-master') else 'local' %}
{%- else %}
{%- set cli = 'unknown' %}
{%- endif %}
{%- do salt['log.debug']('[libsaltcli] the salt command type has been identified to be: ' ~ cli) %}

View File

@ -1,22 +1,204 @@
# -*- coding: utf-8 -*-
# vim: ft=jinja
{#- Start imports as #}
{% import_yaml 'openssh/defaults.yaml' as default_settings %}
{% import_yaml 'openssh/osfamilymap.yaml' as osfamilymap %}
{% import_yaml 'openssh/osmap.yaml' as osmap %}
{% import_yaml 'openssh/osfingermap.yaml' as osfingermap %}
{#- Get the `tplroot` from `tpldir` #}
{%- set tplroot = tpldir.split("/")[0] %}
{%- from tplroot ~ "/libsaltcli.jinja" import cli with context %}
{% set defaults = salt['grains.filter_by'](default_settings,
default='default',
merge=salt['grains.filter_by'](osfamilymap, grain='os_family',
merge=salt['grains.filter_by'](osmap, grain='os',
merge=salt['grains.filter_by'](osfingermap, grain='osfinger')
)
)
{#- Where to lookup parameters source files #}
{%- set map_sources_dir = tplroot | path_join("parameters") %}
{#- Load defaults first to allow per formula default map.jinja configuration #}
{%- set _defaults_filename = map_sources_dir | path_join("defaults.yaml") %}
{%- do salt["log.debug"](
"map.jinja: initialise parameters from "
~ _defaults_filename
) %}
{%- import_yaml _defaults_filename as default_settings %}
{#- List of sources to lookup for parameters #}
{%- do salt["log.debug"]("map.jinja: lookup 'map_jinja' configuration sources") %}
{#- Fallback to previously used grains plus minion `id` #}
{%- set map_sources = [
"osarch",
"os_family",
"os",
"osfinger",
"config_get_lookup",
"config_get",
"id",
] %}
{#- Configure map.jinja from defaults.yaml #}
{%- set map_sources = default_settings | traverse(
"values:map_jinja:sources",
map_sources,
) %}
{#- merge the openssh pillar #}
{% set openssh = salt['pillar.get']('openssh', default=defaults['openssh'], merge=True) %}
{% set ssh_config = salt['pillar.get']('ssh_config', default=defaults['ssh_config'], merge=True) %}
{% set sshd_config = salt['pillar.get']('sshd_config', default=defaults['sshd_config'], merge=True) %}
{#- Lookup global sources #}
{%- set map_sources = salt["config.get"]("map_jinja:sources", map_sources) %}
{#- Lookup per formula sources #}
{%- set map_sources = salt["config.get"](
tplroot ~ ":map_jinja:sources",
map_sources,
) %}
{%- do salt["log.debug"](
"map.jinja: load parameters with sources from "
~ map_sources
) %}
{#- Lookup with `config.get` from configurable roots #}
{%- do salt["log.debug"](
"map.jinja: initialise 'config.get' roots with 'tplroot' "
~ tplroot
) %}
{%- set config_get_roots = [tplroot] %}
{#- Configure `config.get` from defaults.yaml #}
{%- set config_get_roots = default_settings | traverse(
"values:map_jinja:config_get_roots",
config_get_roots
) %}
{#- Lookup global `config.get` roots #}
{%- set config_get_roots = salt["config.get"](
"map_jinja:config_get_roots",
config_get_roots
) %}
{#- Lookup per formula `config.get` roots #}
{%- set config_get_roots = salt["config.get"](
tplroot ~ ":map_jinja:config_get_roots",
config_get_roots,
) %}
{%- do salt["log.debug"](
"map.jinja: load parameters with 'config.get' from roots "
~ config_get_roots
) %}
{#- Work around assignment inside for loop #}
{#- load configuration values used in `config.get` merging strategies #}
{%- set _config = {
"stack": default_settings.get("values", {}),
"merge_strategy": salt["config.get"](tplroot ~ ":strategy", None),
"merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
} %}
{#- the `config.get` merge option only works for `minion` or `local` salt command types #}
{%- if cli in ["minion", "local"] %}
{%- do _config.update(
{
"merge_opt": {"merge": _config["merge_strategy"]},
"merge_msg": ", merge: strategy='" ~ _config["merge_strategy"] ~ "'",
}
) %}
{#- the `config.get` merge option is not available for `ssh` or `unknown` salt command types #}
{%- else %}
{%- if _config["merge_strategy"] %}
{%- do salt["log.error"](
"map.jinja: the 'merge' option of 'config.get' is skipped when the salt command type is '"
~ cli
~ "'"
) %}
{%- endif %}
{%- do _config.update(
{
"merge_opt": {},
"merge_msg": "",
}
) %}
{%- endif %}
{#- process each `map.jinja` source #}
{%- for map_source in map_sources %}
{%- if map_source in ["config_get", "config_get_lookup"] %}
{%- for _config_root in config_get_roots %}
{%- set _config_key = {
"config_get": _config_root,
"config_get_lookup": _config_root ~ ":lookup",
}.get(map_source) %}
{%- do salt["log.debug"](
"map.jinja: retrieve '"
~ _config_key
~ "' with 'config.get'"
~ _config["merge_msg"]
) %}
{%- set _config_get = salt["config.get"](
_config_key, default={}, **_config["merge_opt"]
) %}
{#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
{%- do salt["log.debug"](
"map.jinja: merge '"
~ _config_key
~ "' retrieved with 'config.get'"
~ ", merge: strategy='"
~ _strategy
~ "', lists='"
~ _config["merge_lists"]
~ "'"
) %}
{#- Keep values under each root key when there are more than one #}
{%- if config_get_roots|length > 1 %}
{%- set _config_get = { _config_root: _config_get } %}
{%- endif %}
{%- do _config.update(
{
"stack": salt["slsutil.merge"](
_config["stack"],
_config_get,
strategy=_strategy,
merge_lists=_config["merge_lists"],
)
}
) %}
{%- endfor %}
{%- else %}
{#- Lookup the grain/pillar/... #}
{#- Fallback to use the source name as a direct filename #}
{%- set map_values = salt["config.get"](map_source, []) %}
{#- Mangle `map_source` to use it as literal path #}
{%- if map_values | length == 0 %}
{%- set map_source_parts = map_source.split("/") %}
{%- set map_source = map_source_parts[0:-1] | join("/") %}
{%- set map_values = map_source_parts[-1].rstrip(".yaml") %}
{%- endif %}
{#- Some configuration return list #}
{%- if map_values is string %}
{%- set map_values = [map_values] %}
{%- endif %}
{%- for map_value in map_values %}
{%- set yamlfile = map_sources_dir | path_join(
map_source,
map_value ~ ".yaml",
) %}
{%- do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %}
{%- load_yaml as loaded_values %}
{%- include yamlfile ignore missing %}
{%- endload %}
{%- if loaded_values %}
{#- Merge loaded values on the stack #}
{%- do salt["log.debug"]("map.jinja: merge parameters from " ~ yamlfile) %}
{%- do _config.update(
{
"stack": salt["slsutil.merge"](
_config["stack"],
loaded_values.get("values", {}),
strategy=loaded_values.get("strategy", "smart"),
merge_lists=loaded_values.get("merge_lists", False)
| to_bool,
)
}
) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endfor %}
{%- do salt["log.debug"]("map.jinja: save parameters in variable 'mapdata'") %}
{%- set mapdata = _config["stack"] %}

View File

@ -1,4 +1,5 @@
{% from "openssh/map.jinja" import openssh with context %}
{% from "openssh/map.jinja" import mapdata with context %}
{%- set openssh = mapdata.openssh %}
{% set moduli = salt['pillar.get']('openssh:moduli', False) -%}
{% set moduli_source = salt['pillar.get']('openssh:moduli_source', False) -%}

View File

@ -1,71 +0,0 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
---
Arch:
openssh:
server: openssh
client: openssh
service: sshd
dig_pkg: bind
sshd_config:
Subsystem: sftp /usr/lib/ssh/sftp-server
Debian:
openssh:
server: openssh-server
client: openssh-client
service: ssh
sshd_config:
Subsystem: sftp /usr/lib/openssh/sftp-server
FreeBSD:
openssh:
service: sshd
dig_pkg: bind-tools
sshd_config_group: wheel
ssh_config_group: wheel
sshd_config:
Subsystem: sftp /usr/libexec/sftp-server
Gentoo:
openssh:
server: net-misc/openssh
client: net-misc/openssh
service: sshd
dig_pkg: net-dns/bind-tools
OpenBSD:
openssh:
service: sshd
sshd_config_group: wheel
ssh_config_group: wheel
sshd_config:
Subsystem: sftp /usr/libexec/sftp-server
RedHat:
openssh:
server: openssh-server
client: openssh-clients
service: sshd
dig_pkg: bind-utils
sshd_config:
Subsystem: sftp /usr/libexec/openssh/sftp-server
Solaris:
openssh:
service: network/ssh
sshd_config_group: root
ssh_config_group: root
dig_pkg: bind
sshd_binary: /usr/lib/ssh/sshd
sshd_config:
Subsystem: sftp internal-sftp
Suse:
openssh:
server: openssh
client: openssh
service: sshd
dig_pkg: bind-utils
sshd_config:
Subsystem: sftp /usr/lib/ssh/sftp-server

View File

@ -1,7 +0,0 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
---
Ubuntu-18.04: {}
CentOS-6:
openssh:
host_key_algos: ecdsa,rsa

View File

@ -1,4 +0,0 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
---
FreeBSD: {}

View File

@ -1,7 +1,13 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
---
default:
values:
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
sshd_enable: true
sshd_binary: /usr/sbin/sshd
@ -31,3 +37,4 @@ default:
sshd_config: {}
ssh_config: {}
...

View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == Arch.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
server: openssh
client: openssh
service: sshd
dig_pkg: bind
sshd_config:
Subsystem: sftp /usr/lib/ssh/sftp-server
...

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == Debian.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
server: openssh-server
client: openssh-client
service: ssh
sshd_config:
Subsystem: sftp /usr/lib/openssh/sftp-server
...

View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == FreeBSD.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
service: sshd
dig_pkg: bind-tools
sshd_config_group: wheel
ssh_config_group: wheel
sshd_config:
Subsystem: sftp /usr/libexec/sftp-server
...

View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == Gentoo.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
server: net-misc/openssh
client: net-misc/openssh
service: sshd
dig_pkg: net-dns/bind-tools
...

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == OpenBSD.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
service: sshd
sshd_config_group: wheel
ssh_config_group: wheel
sshd_config:
Subsystem: sftp /usr/libexec/sftp-server
...

View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == RedHat.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
server: openssh-server
client: openssh-clients
service: sshd
dig_pkg: bind-utils
sshd_config:
Subsystem: sftp /usr/libexec/openssh/sftp-server
...

View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == Solaris.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
service: network/ssh
sshd_config_group: root
ssh_config_group: root
dig_pkg: bind
sshd_binary: /usr/lib/ssh/sshd
sshd_config:
Subsystem: sftp internal-sftp
...

View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('os_family') == Suse.
# You just need to add the key:values for this `os_family` that differ
# from `defaults.yaml` + `<osarch>.yaml`.
#
# If you do not need to provide defaults via the `os_family` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
server: openssh
client: openssh
service: sshd
dig_pkg: bind-utils
sshd_config:
Subsystem: sftp /usr/lib/ssh/sftp-server
...

View File

@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
# vim: ft=yaml
#
# Setup variables specific to salt['config.get']('osfinger') == CentOS-6.
# You just need to add the key:values for this `osfinger` that differ
# from `defaults.yaml`.
#
# If you do not need to provide defaults via the `osfinger` config,
# you can remove this file or provide at least an empty dict, e.g.
# values: {}
---
values:
openssh:
host_key_algos: ecdsa,rsa
...

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Amazon Linux AMI-2018
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Amazon Linux-2
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Arch
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# CentOS-6
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# CentOS Linux-7
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# CentOS Linux-8
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Debian-10
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Debian-9
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Fedora-31
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Fedora-32
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Leap-15
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Ubuntu-16.04
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Ubuntu-18.04
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false

View File

@ -1,6 +1,11 @@
# yamllint disable rule:indentation rule:line-length
# Ubuntu-20.04
---
map_jinja:
config_get_roots:
- openssh
- sshd_config
- ssh_config
openssh:
absent_dsa_keys: false
absent_ecdsa_keys: false