From d5262ea5dfb41a5df8247a3616f9fa53357ef919 Mon Sep 17 00:00:00 2001 From: Sammy Date: Thu, 16 May 2019 17:27:21 +1100 Subject: [PATCH] feat(tofs): first implemetation + tplroot First implementation of TOFS for: nginx_config, passenger_config, servers_config and snippets Introduced tplroot on modified files as well Fixed GH link of nginx.conf file in docs/TOFS_pattern.rst Fixed test for snippets name pillar --- docs/TOFS_pattern.rst | 4 +- nginx/config.sls | 18 +++-- nginx/files/{ => default}/nginx.conf | 0 nginx/files/{ => default}/nginx.service | 0 nginx/files/{ => default}/server.conf | 0 nginx/libtofs.jinja | 101 ++++++++++++++++++++++++ nginx/passenger.sls | 9 ++- nginx/servers.sls | 7 +- nginx/servers_config.sls | 24 +++--- nginx/service.sls | 10 ++- nginx/snippets.sls | 11 ++- pillar.example | 45 +++++++++-- 12 files changed, 196 insertions(+), 33 deletions(-) rename nginx/files/{ => default}/nginx.conf (100%) rename nginx/files/{ => default}/nginx.service (100%) rename nginx/files/{ => default}/server.conf (100%) create mode 100644 nginx/libtofs.jinja diff --git a/docs/TOFS_pattern.rst b/docs/TOFS_pattern.rst index 9bf860b..ec3bede 100644 --- a/docs/TOFS_pattern.rst +++ b/docs/TOFS_pattern.rst @@ -39,7 +39,7 @@ Using SaltStack is a simple and effective way to implement configuration managem To avoid this situation we can use the `pillar mechanism `_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja `_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates. -There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas `_' repositories. `Some `_ `developments `_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja `_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer. +There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas `_' repositories. `Some `_ `developments `_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja `_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer. In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values. @@ -432,7 +432,7 @@ The list of ``source_files`` can be given: Resulting in: .. code-block:: sls - + - source: - salt://ntp/files/theminion/etc/ntp.conf.jinja - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja diff --git a/nginx/config.sls b/nginx/config.sls index b1757cb..3cd00b5 100644 --- a/nginx/config.sls +++ b/nginx/config.sls @@ -2,7 +2,9 @@ # # Manages the main nginx server configuration file. -{% from 'nginx/map.jinja' import nginx, sls_block with context %} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %} {% if nginx.install_from_source %} nginx_log_dir: @@ -12,16 +14,18 @@ nginx_log_dir: - group: {{ nginx.server.config.user }} {% endif %} -{% if 'source_path' in nginx.server.config %} -{% set source_path = nginx.server.config.source_path %} -{% else %} -{% set source_path = 'salt://nginx/files/nginx.conf' %} -{% endif %} nginx_config: file.managed: {{ sls_block(nginx.server.opts) }} - name: {{ nginx.lookup.conf_file }} - - source: {{ source_path }} + - source: +{% if 'source_path' in nginx.server.config %} + - {{ nginx.server.config.source_path }} +{% endif %} + {{ files_switch(['nginx.conf'], + 'nginx_config_file_managed' + ) + }} - template: jinja {% if 'source_path' not in nginx.server.config %} - context: diff --git a/nginx/files/nginx.conf b/nginx/files/default/nginx.conf similarity index 100% rename from nginx/files/nginx.conf rename to nginx/files/default/nginx.conf diff --git a/nginx/files/nginx.service b/nginx/files/default/nginx.service similarity index 100% rename from nginx/files/nginx.service rename to nginx/files/default/nginx.service diff --git a/nginx/files/server.conf b/nginx/files/default/server.conf similarity index 100% rename from nginx/files/server.conf rename to nginx/files/default/server.conf diff --git a/nginx/libtofs.jinja b/nginx/libtofs.jinja new file mode 100644 index 0000000..da656a5 --- /dev/null +++ b/nginx/libtofs.jinja @@ -0,0 +1,101 @@ +{%- macro files_switch(source_files, + lookup=None, + default_files_switch=['id', 'os_family'], + indent_width=6, + v1_path_prefix='') %} + {#- + Returns a valid value for the "source" parameter of a "file.managed" + state function. This makes easier the usage of the Template Override and + Files Switch (TOFS) pattern. + + Params: + * source_files: ordered list of files to look for + * lookup: key under ':tofs:source_files' to override + list of source files + * default_files_switch: if there's no config (e.g. pillar) + ':tofs:files_switch' this is the ordered list of grains to + use as selector switch of the directories under + "/files" + * indent_witdh: indentation of the result value to conform to YAML + * v1_path_prefix: (deprecated) only used for injecting a path prefix into + the source, to support older TOFS configs + + Example (based on a `tplroot` of `xxx`): + + If we have a state: + + Deploy configuration: + file.managed: + - name: /etc/yyy/zzz.conf + - source: {{ files_switch(['/etc/yyy/zzz.conf', '/etc/yyy/zzz.conf.jinja'], + lookup='Deploy configuration' + ) }} + - template: jinja + + In a minion with id=theminion and os_family=RedHat, it's going to be + rendered as: + + Deploy configuration: + file.managed: + - name: /etc/yyy/zzz.conf + - source: + - salt://xxx/files/theminion/etc/yyy/zzz.conf + - salt://xxx/files/theminion/etc/yyy/zzz.conf.jinja + - salt://xxx/files/RedHat/etc/yyy/zzz.conf + - salt://xxx/files/RedHat/etc/yyy/zzz.conf.jinja + - salt://xxx/files/default/etc/yyy/zzz.conf + - salt://xxx/files/default/etc/yyy/zzz.conf.jinja + - template: jinja + #} + {#- Get the `tplroot` from `tpldir` #} + {%- set tplroot = tpldir.split('/')[0] %} + {%- set path_prefix = salt['config.get'](tplroot ~ ':tofs:path_prefix', tplroot) %} + {%- set files_dir = salt['config.get'](tplroot ~ ':tofs:dirs:files', 'files') %} + {%- set files_switch_list = salt['config.get']( + tplroot ~ ':tofs:files_switch', + default_files_switch + ) %} + {#- Lookup source_files (v2), files (v1), or fallback to source_files parameter #} + {%- set src_files = salt['config.get']( + tplroot ~ ':tofs:source_files:' ~ lookup, + salt['config.get']( + tplroot ~ ':tofs:files:' ~ lookup, + source_files + ) + ) %} + {#- Only add to [''] when supporting older TOFS implementations #} + {%- set path_prefix_exts = [''] %} + {%- if v1_path_prefix != '' %} + {%- do path_prefix_exts.append(v1_path_prefix) %} + {%- endif %} + {%- for path_prefix_ext in path_prefix_exts %} + {%- set path_prefix_inc_ext = path_prefix ~ path_prefix_ext %} + {#- For older TOFS implementation, use `files_switch` from the config #} + {#- Use the default, new method otherwise #} + {%- set fsl = salt['config.get']( + tplroot ~ path_prefix_ext|replace('/', ':') ~ ':files_switch', + files_switch_list + ) %} + {#- Append an empty value to evaluate as `default` in the loop below #} + {%- if '' not in fsl %} + {%- do fsl.append('') %} + {%- endif %} + {%- for fs in fsl %} + {%- for src_file in src_files %} + {%- if fs %} + {%- set fs_dir = salt['config.get'](fs, fs) %} + {%- else %} + {%- set fs_dir = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %} + {%- endif %} + {%- set url = [ + '- salt:/', + path_prefix_inc_ext.strip('/'), + files_dir.strip('/'), + fs_dir.strip('/'), + src_file.strip('/'), + ] | select | join('/') %} +{{ url | indent(indent_width, true) }} + {%- endfor %} + {%- endfor %} + {%- endfor %} +{%- endmacro %} diff --git a/nginx/passenger.sls b/nginx/passenger.sls index 49fe598..1f96290 100644 --- a/nginx/passenger.sls +++ b/nginx/passenger.sls @@ -3,7 +3,9 @@ # Manages installation of passenger from repo. # Requires install_from_phusionpassenger = True -{% from 'nginx/map.jinja' import nginx, sls_block with context %} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %} {% if salt['grains.get']('os_family') in ['Debian', 'RedHat'] %} include: @@ -27,7 +29,10 @@ passenger_config: file.managed: {{ sls_block(nginx.server.opts) }} - name: {{ nginx.lookup.passenger_config_file }} - - source: salt://nginx/files/nginx.conf + - source: {{ files_switch(['nginx.conf'], + 'passenger_config_file_managed' + ) + }} - template: jinja - context: config: {{ nginx.passenger|json() }} diff --git a/nginx/servers.sls b/nginx/servers.sls index abf6a64..f3033bb 100644 --- a/nginx/servers.sls +++ b/nginx/servers.sls @@ -2,9 +2,10 @@ # # Manages virtual hosts and their relationship to the nginx service. -{% from 'nginx/map.jinja' import nginx, sls_block with context %} -{% from 'nginx/servers_config.sls' import server_states with context %} -{% from 'nginx/service.sls' import service_function with context %} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/servers_config.sls' import server_states with context %} +{%- from tplroot ~ '/service.sls' import service_function with context %} {% macro file_requisites(states) %} {%- for state in states %} diff --git a/nginx/servers_config.sls b/nginx/servers_config.sls index 43000ca..3d0d38c 100644 --- a/nginx/servers_config.sls +++ b/nginx/servers_config.sls @@ -2,7 +2,11 @@ # # Manages the configuration of virtual host files. -{% from 'nginx/map.jinja' import nginx, sls_block with context %} +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %} + {% set server_states = [] %} # Simple path concatenation. @@ -100,17 +104,19 @@ nginx_server_available_dir: file.absent: - name: {{ server_curpath(server) }} {% else %} -{% if settings.config != None and settings.enabled == True %} -{% if 'source_path' in settings.config %} -{% set source_path = settings.config.source_path %} -{% else %} -{% set source_path = 'salt://nginx/files/server.conf' %} -{% endif %} +{% if settings.enabled == True %} {{ conf_state_id }}: file.managed: {{ sls_block(nginx.servers.managed_opts) }} - name: {{ server_curpath(server) }} - - source: {{ source_path }} + - source: +{%- if 'source_path' in settings.config %} + - {{ settings.config.source_path }} +{%- endif %} + {{ files_switch([server, 'server.conf'], + 'server_conf_file_managed' + ) + }} - makedirs: True - template: jinja - require_in: @@ -138,7 +144,7 @@ nginx_server_available_dir: {% else %} {{ manage_status(server, settings.enabled, False) }} {% endif %} -{% if settings.config != None and settings.enabled == True %} +{% if settings.enabled == True %} - require: - file: {{ conf_state_id }} {% endif %} diff --git a/nginx/service.sls b/nginx/service.sls index 5d58fc1..9cbc5cc 100644 --- a/nginx/service.sls +++ b/nginx/service.sls @@ -2,7 +2,10 @@ # # Manages the nginx service. -{% from 'nginx/map.jinja' import nginx, sls_block with context %} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %} + {% set service_function = {True:'running', False:'dead'}.get(nginx.service.enable) %} include: @@ -16,7 +19,10 @@ include: nginx_systemd_service_file: file.managed: - name: /lib/systemd/system/nginx.service - - source: salt://nginx/files/nginx.service + - source: {{ files_switch(['nginx.service'], + 'nginx_systemd_service_file' + ) + }} {% endif %} nginx_service: diff --git a/nginx/snippets.sls b/nginx/snippets.sls index 3ceb2cc..1c57ccb 100644 --- a/nginx/snippets.sls +++ b/nginx/snippets.sls @@ -2,7 +2,9 @@ # # Manages creation of snippets -{% from 'nginx/map.jinja' import nginx, sls_block with context %} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ '/map.jinja' import nginx, sls_block with context %} +{%- from tplroot ~ '/libtofs.jinja' import files_switch with context %} nginx_snippets_dir: file.directory: @@ -12,8 +14,11 @@ nginx_snippets_dir: {% for snippet, config in nginx.snippets.items() %} nginx_snippet_{{ snippet }}: file.managed: - - name: {{ nginx.lookup.snippets_dir }}/{{ snippet }}.conf - - source: salt://nginx/files/server.conf + - name: {{ nginx.lookup.snippets_dir ~ '/' ~ snippet ~ '.conf' }} + - source: {{ files_switch([ snippet, 'server.conf' ], + 'nginx_snippet_file_managed' + ) + }} - template: jinja - context: config: {{ config|json() }} diff --git a/pillar.example b/pillar.example index 887cbc1..4ecc9ec 100644 --- a/pillar.example +++ b/pillar.example @@ -60,6 +60,7 @@ nginx: ##--- --- - - - - - - -- - - - - -- - - --- -- - -- - - - -- - - - - -- - - - -- - - - -- - ## ## You can use snippets to define often repeated configuration once and include it later ## The letsencrypt example below is consumed by "- include: 'snippets/letsencrypt.conf'" + ## Files or Templates can be retrieved by TOFS with snippet name ( Fallback to server.conf ) ##--- --- - - - - - - -- - - -- -- - - --- -- - -- - - - -- - - - - -- - - - -- - - - -- - ## snippets: letsencrypt: @@ -85,7 +86,6 @@ nginx: - server: 127.0.0.1:19999 - keepalive: 64 - server: opts: {} # this partially exposes file.managed parameters as they relate to the main nginx.conf file @@ -93,12 +93,14 @@ nginx: # nginx.conf (main server) declarations # dictionaries map to blocks {} and lists cause the same declaration to repeat with different values # see also http://nginx.org/en/docs/example.html + # Nginx config file or template can be retrieved by TOFS ( Fallback to nginx.conf ) #-- - - - - -- - - -- - - - - -- - - -- - - - -- - - - - - -- - - - - - -- - - - - -- - - - - -- - - # config: include: 'snippets/letsencrypt.conf' - source_path: salt://path_to_nginx_conf_file/nginx.conf # IMPORTANT: This option is mutually exclusive with the rest of the - # options; if it is found other options (worker_processes: 4 and so - # on) are not processed and just upload the file from source + source_path: salt://path_to_nginx_conf_file/nginx.conf # IMPORTANT: This option is mutually exclusive with TOFS and + # the rest of the options; if it is found other options + # (worker_processes: 4 and so on) are not processed + # and just upload the file from source worker_processes: 4 load_module: modules/ngx_http_lua_module.so # pass as very first in configuration; otherwise nginx will fail to start #pid: /var/run/nginx.pid # Directory location must exist (i.e. it's /run/nginx.pid on EL7) @@ -218,7 +220,8 @@ nginx: available_dir: /etc/nginx/sites-available enabled_dir: /etc/nginx/sites-enabled config: - source_path: salt://path-to-site-file/mysite2 + source_path: salt://path-to-site-file/mysite2 # IMPORTANT: This field is mutually exclusive with TOFS + # and other config options, it just uploads the specified file # Below configuration becomes handy if you want to create custom configuration files # for example if you want to create /usr/local/etc/nginx/http_options.conf with @@ -277,7 +280,39 @@ nginx: # Passenger configuration # Default passenger configuration is provided, and will be deployed in # /etc/nginx/conf.d/passenger.conf + # Passenger conf can be retrieved by TOFS ( Fallback to nginx.conf ) passenger: passenger_root: /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini passenger_ruby: /usr/bin/ruby passenger_instance_registry_dir: /var/run/passenger-instreg + + tofs: + # The files_switch key serves as a selector for alternative + # directories under the formula files directory. See TOFS pattern + # doc for more info. + # Note: Any value not evaluated by `config.get` will be used literally. + # This can be used to set custom paths, as many levels deep as required. + # files_switch: + # - any/path/can/be/used/here + # - id + # - role + # - osfinger + # - os + # - os_family + # All aspects of path/file resolution are customisable using the options below. + # This is unnecessary in most cases; there are sensible defaults. + # path_prefix: template_alt + # dirs: + # files: files_alt + # default: default_alt + source_files: + nginx_config_file_managed: + - alt_nginx.conf + passenger_config_file_managed: + - alt_nginx.conf + server_conf_file_managed: + - alt_server.conf + nginx_systemd_service_file: + - alt_nginx.service + nginx_snippet_file_managed: + - alt_server.conf