Merge pull request #435 from myii/chore/standardise-structure

feat: use `semantic-release` cross-formula standard structure

.gitignore

@@ -48,6 +48,7 @@ coverage.xml
 .kitchen
 .kitchen.local.yml
 kitchen.local.yml
+junit-*.xml
 
 # Translations
 *.mo
@@ -110,5 +111,12 @@ Gemfile.lock
 docs/*.md
 
 # Vim
-*.swp
+*.sw?
-*.swo
+
+## Collected when centralising formulas (check and sort)
+# `collectd-formula`
+.pytest_cache/
+/.idea/
+Dockerfile.*_*
+ignore/
+tmp/

.travis.yml

@@ -1,6 +1,10 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+dist: bionic
 stages:
   - test
-  - commitlint
+  - lint
   - name: release
     if: branch = master AND type != pull_request
 
@@ -15,36 +19,57 @@ services:
 # the `platforms` defined in `kitchen.yml`
 env:
   matrix:
-    - INSTANCE: v2019-2-py3-debian-9
+    # The ordering used below has been selected based on the time required in Travis
-    - INSTANCE: v2019-2-py3-ubuntu-1804
+    # The slower ones are kept as high up as possible, to run concurrently rather than
-    # - INSTANCE: v2019-2-py2-centos-7
+    # slow down the entire run at the end (i.e. `centos-6` and `opensuse`)
-    - INSTANCE: v2019-2-py2-fedora-29
+    # However, the groupings needed to be maintained in some semblance of order
+    # so this is a best-effort matrix, in the circumstances
 
-    - INSTANCE: v2018-3-py2-debian-8
+    # develop-py3
-    - INSTANCE: v2018-3-py2-ubuntu-1604
+    - INSTANCE: develop-py3-debian-9
-    - INSTANCE: v2018-3-py2-bootstrap-centos-6
+    - INSTANCE: develop-py3-opensuse-150
-    - INSTANCE: v2018-3-py2-forced-version-fedora-28
+    # - INSTANCE: develop-py3-ubuntu-1804
-    # - INSTANCE: v2018-3-py2-opensuse-423
 
-    - INSTANCE: v2017-7-py2-debian-8
+    # 2017.7-py2
-    - INSTANCE: v2017-7-py2-ubuntu-1604
     - INSTANCE: v2017-7-py2-bootstrap-centos-6
+    # - INSTANCE: v2017-7-py2-debian-8
+    - INSTANCE: v2017-7-py2-ubuntu-1604
+
+    # 2018.3-py2
+    # - INSTANCE: v2018-3-py2-centos-7
+    - INSTANCE: v2018-3-py2-debian-8
+    - INSTANCE: v2018-3-py2-forced-version-fedora-29
+    # - INSTANCE: v2018-3-py2-ubuntu-1604
+
+    # 2019.2-py2
+    - INSTANCE: v2019-2-py2-centos-7
+    - INSTANCE: v2019-2-py2-fedora-30
+
+    # 2019.2-py3
+    # - INSTANCE: v2019-2-py3-debian-9
+    - INSTANCE: v2019-2-py3-opensuse-150
+    - INSTANCE: v2019-2-py3-ubuntu-1804
 
 script:
-  - bundle exec kitchen verify ${INSTANCE}
+  - bin/kitchen verify ${INSTANCE}
 
 jobs:
   include:
-    # Define the commitlint stage
+    # Define the `lint` stage (runs `yamllint` and `commitlint`)
-    - stage: commitlint
+    - stage: lint
       language: node_js
       node_js: lts/*
       before_install: skip
       script:
+        # Install and run `yamllint`
+        # Need at least `v1.17.0` for the `yaml-files` setting
+        - pip install --user yamllint>=1.17.0
+        - yamllint -s .
+        # Install and run `commitlint`
         - npm install @commitlint/config-conventional -D
         - npm install @commitlint/travis-cli -D
         - commitlint-travis
-    # Define the release stage that runs semantic-release
+    # Define the release stage that runs `semantic-release`
     - stage: release
       language: node_js
       node_js: lts/*

.yamllint

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+# Extend the `default` configuration provided by `yamllint`
+extends: default
+
+# Files to ignore completely
+# 1. All YAML files under directory `node_modules/`, introduced during the Travis run
+# 2. Any SLS files under directory `test/`, which are actually state files
+# 3. All YAML files heavily reliant on Jinja; these can be tackled in a subsequent PR
+ignore: |
+  node_modules/
+  test/**/states/**/*.sls
+  salt/osfamilymap.yaml
+  salt/osmap.yaml
+
+yaml-files:
+  # Default settings
+  - '*.yaml'
+  - '*.yml'
+  - .yamllint
+  # SaltStack Formulas additional settings
+  - '*.example'
+  - test/**/*.sls
+
+rules:
+  empty-values:
+    forbid-in-block-mappings: true
+    forbid-in-flow-mappings: true
+  line-length:
+    # Increase from default of `80`
+    # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`)
+    max: 88

FORMULA

@@ -6,3 +6,4 @@ release: 1
 minimum_version: 2015.8
 summary: Formula for install Saltstack
 description: Formula for installing Saltstack
+top_level_dir: salt

bin/kitchen

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'kitchen' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("test-kitchen", "kitchen")

docs/CONTRIBUTING.rst

@@ -157,5 +157,3 @@ An example of that:
    `template package`, this formula no longer supports the installation of
    packages.
 
-
-

docs/README.rst

@@ -205,28 +205,42 @@ Testing
 
 Linux testing is done with ``kitchen-salt``.
 
-``kitchen converge``
+Requirements
-^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^
 
-Creates the docker instance and runs the ``template`` main state, ready for testing.
+* Ruby
+* Docker
 
-``kitchen verify``
+.. code-block:: bash
-^^^^^^^^^^^^^^^^^^
+
+   $ gem install bundler
+   $ bundle install
+   $ bin/kitchen test [platform]
+
+Where ``[platform]`` is the platform name defined in ``kitchen.yml``,
+e.g. ``debian-9-2019-2-py3``.
+
+``bin/kitchen converge``
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Creates the docker instance and runs the ``salt`` main states, ready for testing.
+
+``bin/kitchen verify``
+^^^^^^^^^^^^^^^^^^^^^^
 
 Runs the ``inspec`` tests on the actual instance.
 
-``kitchen destroy``
+``bin/kitchen destroy``
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^
 
 Removes the docker instance.
 
-``kitchen test``
+``bin/kitchen test``
-^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^
 
 Runs all of the stages above in one go: i.e. ``destroy`` + ``converge`` + ``verify`` + ``destroy``.
 
-``kitchen login``
+``bin/kitchen login``
-^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^
 
 Gives you SSH access to the instance for manual testing.
-

docs/TOFS_pattern.rst

@@ -39,7 +39,7 @@ Using SaltStack is a simple and effective way to implement configuration managem
 
 To avoid this situation we can use the `pillar mechanism <http://docs.saltstack.com/en/latest/topics/pillar/>`_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja <http://docs.saltstack.com/en/latest/topics/tutorials/pillar.html>`_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates.
 
-There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/spsoit/nginx-formula/blob/81de880fe0276dd9488ffa15bc78944c0fc2b919/nginx/ng/files/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
+There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/saltstack-formulas/nginx-formula/blob/f74254c07e188bd448eaf1c5f9c802d78c4c005e/nginx/files/default/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
 
 In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values.
 
@@ -325,6 +325,7 @@ We can simplify the ``conf.sls`` with the new ``files_switch`` macro to use in t
 
 
 * This uses ``config.get``, searching for ``ntp:tofs:source_files:Configure NTP`` to determine the list of template files to use.
+* If this returns a result, the default of ``['/etc/ntp.conf.jinja']`` will be appended to it.
 * If this does not yield any results, the default of ``['/etc/ntp.conf.jinja']`` will be used.
 
 In ``libtofs.jinja``, we define this new macro ``files_switch``.
@@ -426,7 +427,6 @@ The list of ``source_files`` can be given:
      tofs:
        source_files:
          Configure NTP:
-           - '/etc/ntp.conf.jinja'
            - '/etc/ntp.conf_alt.jinja'
 
 Resulting in:
@@ -434,10 +434,85 @@ Resulting in:
 .. code-block:: sls
 
          - source:
-           - salt://ntp/files/theminion/etc/ntp.conf.jinja
            - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja
-           - salt://ntp/files/Debian/etc/ntp.conf.jinja
+           - salt://ntp/files/theminion/etc/ntp.conf.jinja
            - salt://ntp/files/Debian/etc/ntp.conf_alt.jinja
-           - salt://ntp/files/default/etc/ntp.conf.jinja
+           - salt://ntp/files/Debian/etc/ntp.conf.jinja
            - salt://ntp/files/default/etc/ntp.conf_alt.jinja
+           - salt://ntp/files/default/etc/ntp.conf.jinja
 
+Note: This does *not* override the default value.
+Rather, the value from the pillar/config is prepended to the default.
+
+Using sub-directories for ``components``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If your formula is composed of several components, you may prefer to provides files under sub-directories, like in the `systemd-formula <https://github.com/saltstack-formulas/systemd-formula>`_.
+
+.. code-block::
+
+   /srv/saltstack/systemd-formula/
+     systemd/
+       init.sls
+       libtofs.jinja
+       map.jinja
+       networkd/
+         init.sls
+         files/
+           default/
+             network/
+               99-default.link
+       resolved/
+         init.sls
+         files/
+           default/
+             resolved.conf
+       timesyncd/
+         init.sls
+         files/
+           Arch/
+             resolved.conf
+           Debian/
+             resolved.conf
+           default/
+             resolved.conf
+           Ubuntu/
+             resolved.conf
+
+For example, the following ``formula.component.config`` SLS:
+
+.. code-block:: sls
+
+   {%- from "formula/libtofs.jinja" import files_switch with context %}
+
+   formula configuration file:
+     file.managed:
+       - name: /etc/formula.conf
+       - user: root
+       - group: root
+       - mode: 644
+       - template: jinja
+       - source: {{ files_switch(['formula.conf'],
+                                 lookup='formula',
+                                 use_subpath=True
+                    )
+                 }}
+
+will be rendered on a ``Debian`` minion named ``salt-formula.ci.local`` as:
+
+.. code-block:: sls
+
+   formula configuration file:
+     file.managed:
+       - name: /etc/formula.conf
+       - user: root
+       - group: root
+       - mode: 644
+       - template: jinja
+       - source:
+         - salt://formula/component/files/salt-formula.ci.local/formula.conf
+         - salt://formula/component/files/Debian/formula.conf
+         - salt://formula/component/files/default/formula.conf
+         - salt://formula/files/salt-formula.ci.local/formula.conf
+         - salt://formula/files/Debian/formula.conf
+         - salt://formula/files/default/formula.conf

kitchen.yml

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
 ---
 driver:
   name: docker
@@ -11,17 +13,22 @@ platforms:
   - name: debian-9
   - name: ubuntu-18.04
   - name: centos-7
-  - name: fedora-29
+  - name: fedora-30
-  - name: opensuse-42.3
+  - name: opensuse-15.0
     driver:
+      image: opensuse/leap:15.0
       run_command: /usr/lib/systemd/systemd
       provision_command:
         - systemctl enable sshd.service
+    # Workaround to avoid intermittent failures on `opensuse-leap-15`:
+    # => SCP did not finish successfully (255):  (Net::SCP::Error)
+    transport:
+      max_ssh_sessions: 1
 
   # Previous distros
   - name: debian-8
   - name: ubuntu-16.04
-  - name: fedora-28
+  - name: fedora-29
   # centos-6 guest fails on Debian hosts due to vsyscall issues, see
   # https://hub.docker.com/_/centos, "A note about vsyscall"
   - name: centos-6
@@ -43,55 +50,58 @@ provisioner:
         - salt.pkgrepo
         - salt.master
         - salt.minion
+
+verifier:
+  # https://www.inspec.io/
+  name: inspec
+  sudo: true
+  # cli, documentation, html, progress, json, json-min, json-rspec, junit
+  reporter:
+    - cli
+
+suites:
+  # Latest distros, salt develop, python3
+  # These distros have py3 packages available in salt's repo
+  - name: develop-py3
+    includes:
+      - debian-9
+      - ubuntu-18.04
+      - opensuse-15.0
+    provisioner:
+      salt_bootstrap_options: -X -x python3 -d git %s
+      salt_version: 'develop'
       pillars:
         top.sls:
           base:
             '*':
               - salt
-    salt.sls:
+              - develop-py3
-      salt:
+      pillars_from_files:
-        master:
+        salt.sls: test/salt/pillar/salt.sls
-          fileserver_backend:
+        develop-py3.sls: test/salt/pillar/develop-py3.sls
-            - rootfs
+    verifier:
-          file_roots:
+      inspec_tests:
-            base:
+        - path: test/integration/develop
-              - /srv/salt
-          pillar_roots:
-            base:
-              - /srv/pillar
-        minion:
-          master: localhost
-          fileserver_backend:
-            - rootfs
-          file_roots:
-            base:
-              - /srv/salt
-          pillar_roots:
-            base:
-              - /srv/pillar
 
-verifier:
-  name: inspec
-  sudo: true
-  reporter:
-    - cli
-
-suites:
   # Latest distros, latest salt, python3
   # These distros have py3 packages available in salt's repo
   - name: v2019-2-py3
     includes:
       - debian-9
       - ubuntu-18.04
+      - opensuse-15.0
     provisioner:
       salt_bootstrap_options: -X -x python3 -d git %s
       salt_version: '2019.2'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2019.2'
+            '*':
-            py_ver: 'py3'
+              - salt
-
+              - v2019-2-py3
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2019-2-py3.sls: test/salt/pillar/v2019-2-py3.sls
     verifier:
       inspec_tests:
         - path: test/integration/2019-2
@@ -101,15 +111,18 @@ suites:
   - name: v2019-2-py2
     includes:
       - centos-7
-      - fedora-29
+      - fedora-30
     provisioner:
       salt_version: '2019.2'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2019.2'
+            '*':
-            py_ver: 'py2'
+              - salt
-
+              - v2019-2-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2019-2-py2.sls: test/salt/pillar/v2019-2-py2.sls
     verifier:
       inspec_tests:
         - path: test/integration/2019-2
@@ -119,53 +132,42 @@ suites:
     includes:
       - debian-8
       - ubuntu-16.04
-      - opensuse-42.3
+      - centos-7
     provisioner:
       # We require an old version of salt in the provisioner or,
       # the salt formula fails to downgrade to the desired version to test
       salt_version: '2018.3'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2018.3'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2018-3-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2018-3-py2.sls: test/salt/pillar/v2018-3-py2.sls
     verifier:
       inspec_tests:
         - path: test/integration/2018-3
 
-  # centos-6 ships with python2.6, so it requires extra bootstrapping parameters
+  # To test fedora 29 & salt v2018.3, we need to force the package version
-  # to install python2.7
-  - name: v2018-3-py2-bootstrap
-    includes:
-      - centos-6
-    provisioner:
-      salt_bootstrap_options: -X -d stable %s
-      salt_version: '2018.3'
-      pillars:
-        salt.sls:
-          salt:
-            release: '2018.3'
-            py_ver: 'py2'
-    verifier:
-      inspec_tests:
-        - path: test/integration/2018-3
-
-  # To tests fedora 28 & salt v2018.2, we need to force the package version
   # otherwise the image, which includes the 'updates' repo, will install 2019.2
   - name: v2018-3-py2-forced-version
     includes:
-      - fedora-28
+      - fedora-29
     provisioner:
       # We require an old version of salt in the provisioner or,
       # the salt formula fails to downgrade to the desired version to test
       salt_version: '2018.3'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2018.3'
+            '*':
-            py_ver: 'py2'
+              - salt
-            version: '2018.3.0-1.fc28'
+              - v2018-3-py2-forced-version
-
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2018-3-py2-forced-version.sls: test/salt/pillar/v2018-3-py2-forced-version.sls
     verifier:
       inspec_tests:
         - path: test/integration/2018-3
@@ -180,10 +182,14 @@ suites:
       # the salt formula fails to downgrade to the desired version to test
       salt_version: '2017.7'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2017.7'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2017-7-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2017-7-py2.sls: test/salt/pillar/v2017-7-py2.sls
     verifier:
       inspec_tests:
         - path: test/integration/2017-7
@@ -198,11 +204,14 @@ suites:
       salt_bootstrap_options: -X -d stable %s
       salt_version: '2017.7'
       pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2017.7'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2017-7-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2017-7-py2.sls: test/salt/pillar/v2017-7-py2.sls
     verifier:
       inspec_tests:
         - path: test/integration/2017-7
-

pillar.example

@@ -1,24 +1,27 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
 salt:
   # Set this to true to clean any non-salt-formula managed files out of
   # /etc/salt/{master,minion}.d ... You really don't want to do this on 2015.2
   # and up as it'll wipe out important files that Salt relies on.
-  clean_config_d_dir: False
+  clean_config_d_dir: false
 
   # This state will remove "/etc/salt/minion" when you set this to true.
-  minion_remove_config: True
+  minion_remove_config: true
 
   # This state will remove "/etc/salt/master" when you set this to true.
-  master_remove_config: True
+  master_remove_config: true
 
   # Set this to 'py3' to install the Python 3 packages.
   # If this is not set, the Python 2 packages will be installed by default.
   py_ver: 'py3'
 
-  # Set this to False to not have the formula install packages (in the case you
+  # Set this to false to not have the formula install packages (in the case you
   # install Salt via git/pip/etc.)
-  install_packages: True
+  install_packages: true
 
-  # Optional: set salt version (if install_packages is set to True)
+  # Optional: set salt version (if install_packages is set to true)
   version: 2017.7.2-1.el7
 
   # to overwrite map.jinja salt packages
@@ -37,14 +40,18 @@ salt:
   release: '2018.3'
 
   # MacOS has no package management.
-  # Instead, we use file.managed to download an appropriate .pkg file and macpackage.installed to install it
+  # Instead, we use file.managed to download an appropriate .pkg file and
-  # 'version', if set (see above), will be used to check the .pkg version to determine if it should be installed
+  # macpackage.installed to install it 'version', if set (see above), will be
+  # used to check the .pkg version to determine if it should be installed
   #
-  # NOTE: if 'version' is not set version comparison will not occur and the .pkg WILL NOT be installed if a salt
+  # NOTE: if 'version' is not set version comparison will not occur and the
-  # .pkg is already installed
+  # .pkg WILL NOT be installed if a salt .pkg is already installed
-  # NOTE: salt_minion_pkg_hash, if set, will be passed into file.managed's source_hash, use URL or hash string
+  # NOTE: salt_minion_pkg_hash, if set, will be passed into file.managed's
+  # source_hash, use URL or hash string
+  # yamllint disable rule:line-length
   salt_minion_pkg_source: 'https://repo.saltstack.com/osx/salt-2017.7.4-py3-x86_64.pkg'
   salt_minion_pkg_hash: 'https://repo.saltstack.com/osx/salt-2017.7.4-py3-x86_64.pkg.md5'
+  # yamllint enable rule:line-length
 
   # tofs:
   #   The files_switch key serves as a selector for alternative
@@ -71,7 +78,7 @@ salt:
   #       - 'alt_minion.d'
 
   # salt master config
-  master_config_use_TOFS: True
+  master_config_use_TOFS: true
   master:
     fileserver_backend:
       - git
@@ -98,8 +105,9 @@ salt:
       port: 8000
       ssl_crt: /etc/pki/api/certs/server.crt
       ssl_key: /etc/pki/api/certs/server.key
-      debug: False
+      debug: false
-      disable_ssl: False
+      disable_ssl: false
+      # yamllint disable-line rule:line-length
     # for profile configuration as https://docs.saltstack.com/en/latest/topics/tutorials/lxc.html#tutorial-lxc-profiles
     lxc.container_profile:
       debian:
@@ -118,10 +126,10 @@ salt:
           type: veth
           flags: up
     ## for external auth - LDAP
-      # filter to use for Active Directory LDAP
+    ## filter to use for Active Directory LDAP
-    auth.ldap.filter: {% raw %}'sAMAccountName={{username}}'{% endraw %}
+    # auth.ldap.filter: {% raw %}'sAMAccountName={{username}}'{% endraw %}
-      # filter to use for Most other LDAP servers
+    ## filter to use for Most other LDAP servers
-    auth.ldap.filter: {% raw %}'uid={{ username }}'{% endraw %}
+    # auth.ldap.filter: {% raw %}'uid={{ username }}'{% endraw %}
 
     # Define winrepo provider, by default support order is pygit2, gitpython
     # Set to gitpython for Debian & Ubuntu to get around saltstack/salt#35993
@@ -132,7 +140,7 @@ salt:
     engines:
       - slack:
           token: xoxp-XXXXX-XXXXXXX   # use Slack's legacy API token
-          control: True
+          control: true
           valid_users:
             - someuser
             - otheruser
@@ -152,16 +160,16 @@ salt:
           - /srv/salt/reactors/deploy.sls
 
   # salt minion config:
-  minion_config_use_TOFS: True
+  minion_config_use_TOFS: true
   minion:
 
     # single master setup
     master: salt
 
     # multi master setup
-    #master:
+    # master:
-      #- salt_master_1
+    #   - salt_master_1
-      #- salt_master_2
+    #   - salt_master_2
 
     fileserver_backend:
       - git
@@ -176,7 +184,7 @@ salt:
       base:
         - /srv/pillar
     module_config:
-      test: True
+      test: true
       test.foo: foo
       test.bar:
         - baz
@@ -209,7 +217,7 @@ salt:
     engines:
       - slack:
           token: xoxp-XXXXX-XXXXXXX   # use Slack's legacy API token
-          control: True
+          control: true
           valid_users:
             - someuser
             - otheruser
@@ -248,14 +256,14 @@ salt:
       hosts:
         - example.elasticsearch.host:9200
         - example.elasticsearch.host2:9200
-      index_date: True
+      index_date: true
       index: salt
       number_of_shards: 5
       number_of_replicas: 2
-      debug_returner_payload: True
+      debug_returner_payload: true
-      states_count: True
+      states_count: true
-      states_order_output: True
+      states_order_output: true
-      states_single_index: True
+      states_single_index: true
       functions_blacklist:
         - test.ping
         - saltutil.find_job
@@ -307,6 +315,7 @@ salt:
     aws_key: AWSKEYIJSHJAIJS6JSH
     aws_secret: AWSSECRETYkkDY1iQf9zRtl9+pW+Nm+aZY95
     gce_project: test
+    # yamllint disable-line rule:line-length
     gce_service_account_email_address: 867543072364-orl4h2tpp8jcn1tr9ipj@developer.gserviceaccount.com
     rsos_user: afeawofghob
     rsos_tenant: tenant_id_number
@@ -322,12 +331,13 @@ salt:
     prod1:
       host: host.example.com
       user: ubuntu
-      sudo: True
+      sudo: true
       priv: /etc/salt/ssh_keys/sshkey.pem
   gitfs:
     keys:
       global:
-        # key and pub end up being the extension used on the key file. values other than key and pub are possible
+        # key and pub end up being the extension used on the key file
+        # values other than key and pub are possible
         key: |
           -----BEGIN RSA PRIVATE KEY-----
           ...........
@@ -364,8 +374,8 @@ salt_formulas:
       baseurl: https://github.com/saltstack-formulas
       # Directory where Git repositories are downloaded
       basedir: /srv/formulas
-      # Update the git repository to the latest version (False by default)
+      # Update the git repository to the latest version (false by default)
-      update: False
+      update: false
       # Options passed directly to the git.latest state
       options:
         rev: master
@@ -373,7 +383,7 @@ salt_formulas:
         identity: /path/to/.ssh/id_rsa_github_username
     dev:
       basedir: /srv/formulas/dev
-      update: True
+      update: true
       options:
         rev: develop
     # Alternatively, a single directory with multiple branches can be used
@@ -394,21 +404,21 @@ salt_formulas:
         rev: staging
     upstream:
       baseurl: git@github.com:saltstack-formulas
-      update: True
+      update: true
       options:
         branch: upstream
         remote: upstream
   # Options of the file.directory state that creates the directory where
   # the git repositories of the formulas are stored
   basedir_opts:
-    makedirs: True
+    makedirs: true
     user: root
     group: root
     mode: 755
   # Explicitly checkout the original branch for repos after the
-  # git.latest states have been processed (False by default)
+  # git.latest states have been processed (false by default)
   # Enable if using the alternative method (single directory, multiple branches)
-  checkout_orig_branch: True
+  checkout_orig_branch: true
   # List of formulas to enable in each environment
   list:
     base:
@@ -430,4 +440,3 @@ salt_formulas:
       - salt-formula
       - postfix-formula
       - openssh-formula
-

salt/defaults.yaml

@@ -1,20 +1,21 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 salt:
   version: ''
   rootuser: root
   rootgroup: root
-  install_packages: True
+  install_packages: true
-  use_pip: False
+  use_pip: false
-  clean_config_d_dir: True
+  clean_config_d_dir: true
-  restart_via_at: False
+  restart_via_at: false
 
   config_path: /etc/salt
 
-  minion_remove_config: False
+  minion_remove_config: false
-  master_remove_config: False
+  master_remove_config: false
-  minion_config_use_TOFS: False
+  minion_config_use_TOFS: false
-  master_config_use_TOFS: False
+  master_config_use_TOFS: false
 
   minion_service: salt-minion
   master_service: salt-master
@@ -36,22 +37,22 @@ salt:
 
   gitfs:
     dulwich:
-      install_from_source: True
+      install_from_source: true
     pygit2:
-      install_from_source: True
+      install_from_source: true
       version: 0.23.0
       git:
         # if not false, should be state name
-        require_state: False
+        require_state: false
         install_from_package: git
       libgit2:
         version: 0.23.0
-        install_from_source: True
+        install_from_source: true
         build_parent_dir: /usr/src/
         # hash necessary until github issue #9272 is addressed
         download_hash: 683d1164e361e2a0a8d52652840e2340
     gitpython:
-      install_from_source: False
+      install_from_source: false
 
   cloud:
     template_sources:
@@ -60,12 +61,12 @@ salt:
       maps: salt://salt/files/cloud.maps.d
 
 salt_formulas:
-  checkout_orig_branch: False
+  checkout_orig_branch: false
   git_opts:
     default:
       baseurl: https://github.com/saltstack-formulas
       basedir: /srv/formulas
-      update: False
+      update: false
       options:
         rev: master
         output_loglevel: 'quiet'

salt/libtofs.jinja

@@ -2,7 +2,7 @@
                        lookup=None,
                        default_files_switch=['id', 'os_family'],
                        indent_width=6,
-                       v1_path_prefix='') %}
+                       use_subpath=False) %}
   {#-
     Returns a valid value for the "source" parameter of a "file.managed"
     state function. This makes easier the usage of the Template Override and
@@ -10,15 +10,15 @@
 
     Params:
       * source_files: ordered list of files to look for
-      * lookup: key under '<tplroot>:tofs:source_files' to override
+      * lookup: key under '<tplroot>:tofs:source_files' to prepend to the
         list of source files
       * default_files_switch: if there's no config (e.g. pillar)
         '<tplroot>:tofs:files_switch' this is the ordered list of grains to
         use as selector switch of the directories under
         "<path_prefix>/files"
       * indent_witdh: indentation of the result value to conform to YAML
-      * v1_path_prefix: (deprecated) only used for injecting a path prefix into
+      * use_subpath: defaults to `False` but if set, lookup the source file
-        the source, to support older TOFS configs
+        recursively from the current state directory up to `tplroot`
 
     Example (based on a `tplroot` of `xxx`):
 
@@ -55,20 +55,24 @@
       tplroot ~ ':tofs:files_switch',
       default_files_switch
   ) %}
-  {#- Lookup source_files (v2), files (v1), or fallback to source_files parameter #}
+  {#- Lookup source_files (v2), files (v1), or fallback to an empty list #}
   {%- set src_files = salt['config.get'](
       tplroot ~ ':tofs:source_files:' ~ lookup,
-      salt['config.get'](
+      salt['config.get'](tplroot ~ ':tofs:files:' ~ lookup, [])
-          tplroot ~ ':tofs:files:' ~ lookup,
-          source_files
-      )
   ) %}
+  {#- Append the default source_files #}
+  {%- set src_files = src_files + source_files %}
   {#- Only add to [''] when supporting older TOFS implementations #}
   {%- set path_prefix_exts = [''] %}
-  {%- if v1_path_prefix != '' %}
+  {%- if use_subpath and tplroot != tpldir %}
-    {%- do path_prefix_exts.append(v1_path_prefix) %}
+    {#- Walk directory tree to find {{ files_dir }} #}
+    {%- set subpath_parts = tpldir.lstrip(tplroot).lstrip('/').split('/') %}
+    {%- for path in subpath_parts %}
+      {%- set subpath = subpath_parts[0:loop.index] | join('/') %}
+      {%- do path_prefix_exts.append('/' ~ subpath) %}
+    {%- endfor %}
   {%- endif %}
-  {%- for path_prefix_ext in path_prefix_exts %}
+  {%- for path_prefix_ext in path_prefix_exts|reverse %}
     {%- set path_prefix_inc_ext = path_prefix ~ path_prefix_ext %}
     {#- For older TOFS implementation, use `files_switch` from the config #}
     {#- Use the default, new method otherwise #}
@@ -83,10 +87,16 @@
     {%- for fs in fsl %}
       {%- for src_file in src_files %}
         {%- if fs %}
-          {%- set fs_dir = salt['config.get'](fs, fs) %}
+          {%- set fs_dirs = salt['config.get'](fs, fs) %}
         {%- else %}
-          {%- set fs_dir = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
+          {%- set fs_dirs = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
         {%- endif %}
+        {#- Force the `config.get` lookup result as a list where necessary #}
+        {#- since we need to also handle grains that are lists #}
+        {%- if fs_dirs is string %}
+          {%- set fs_dirs = [fs_dirs] %}
+        {%- endif %}
+        {%- for fs_dir in fs_dirs %}
           {%- set url = [
               '- salt:/',
               path_prefix_inc_ext.strip('/'),
@@ -98,4 +108,5 @@
         {%- endfor %}
       {%- endfor %}
     {%- endfor %}
+  {%- endfor %}
 {%- endmacro %}

salt/osfamilymap.yaml

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 
 {% import_yaml "salt/ospyvermap.yaml" as ospyvermap %}
 {% set ospyver = salt['grains.filter_by'](ospyvermap, grain='os_family') or {} %}
@@ -27,13 +28,13 @@ Debian:
   pyinotify: python-pyinotify
   gitfs:
     pygit2:
-      install_from_source: True
+      install_from_source: true
       version: 0.22.1
       git:
-        require_state: False
+        require_state: false
         install_from_package: git
       libgit2:
-        install_from_source: False
+        install_from_source: false
 
 RedHat:
   pkgrepo: 'https://repo.saltstack.com/{{ py_ver_repr }}/redhat/$releasever/$basearch/{{ salt_release }}'
@@ -42,11 +43,11 @@ RedHat:
   python_git: GitPython
   gitfs:
     gitpython:
-      install_from_source: False
+      install_from_source: false
     pygit2:
-      install_from_source: False
+      install_from_source: false
       git:
-        require_state: False
+        require_state: false
         install_from_package: git
   master:
     gitfs_provider: pygit2
@@ -58,9 +59,9 @@ Suse:
   pyinotify: python-pyinotify
   gitfs:
     pygit2:
-      install_from_source: False
+      install_from_source: false
       git:
-        require_state: False
+        require_state: false
         install_from_package: git
   master:
     gitfs_provider: pygit2

salt/osmap.yaml

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 
 {% import_yaml "salt/ospyvermap.yaml" as ospyvermap %}
 {% set ospyver = salt['grains.filter_by'](ospyvermap, grain='os_family') or {} %}
@@ -24,9 +25,9 @@ Ubuntu:
   pygit2: python-pygit2
   gitfs:
     pygit2:
-      install_from_source: False
+      install_from_source: false
       git:
-        require_state: False
+        require_state: false
         install_from_package: Null
 
 Raspbian:
@@ -46,7 +47,7 @@ SmartOS:
   python_dulwich: 'py27-dulwich'
   gitfs:
     dulwich:
-      install_from_source: False
+      install_from_source: false
   config_path: /opt/local/etc/salt
   master:
     gitfs_provider: dulwich

salt/ospyvermap.yaml

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
-
+---
 Debian:
   py2: 'apt'
   py3: 'py3'

salt/pkgrepo/suse/install.sls

@@ -5,7 +5,7 @@
 salt-pkgrepo-install-saltstack-suse:
   pkgrepo.managed:
     - name: systemsmanagement_saltstack_products
-    - humanname: SaltStack repo for Opensuse 42.3
+    - humanname: SaltStack repo for Opensuse 15.x
     - baseurl: {{ salt_settings.pkgrepo }}
     - enabled: 1
     - gpgcheck: 1

test/integration/2018-3/pkgs_spec.rb

@@ -1,12 +1,10 @@
 case os[:name]
 when 'centos'
-  version = '2018.3.4-1.el6'
+  version = '2018.3.4-1.el7'
 when 'debian', 'ubuntu'
   version = '2018.3.4+ds-1'
-when 'opensuse'
-  version = '2018.3.0-42.1'
 when 'fedora'
-  version = '2018.3.0-1.fc28'
+  version = '2018.3.2-5.fc29'
 end
 
 control 'salt packages' do

test/integration/2019-2/pkgs_spec.rb

@@ -2,7 +2,9 @@ case os[:name]
 when 'centos'
   version = '2019.2.0-1.el7'
 when 'fedora'
-  version = '2019.2.0-1.fc29'
+  version = '2019.2.0-1.fc30'
+when 'opensuse'
+  version = '2019.2.0-lp150.3.33.1'
 when 'debian', 'ubuntu'
   version = '2019.2.0+ds-1'
 end

test/integration/develop/pkgs_spec.rb

@@ -0,0 +1,12 @@
+control 'salt packages' do
+  title 'should be installed'
+
+  %w(
+    salt-master
+    salt-minion
+  ).each do |p|
+    describe package(p) do
+      it { should be_installed }
+    end
+  end
+end

test/integration/develop/service_spec.rb

@@ -0,0 +1,13 @@
+control 'salt services' do
+  title 'should be running'
+
+  %w(
+    salt-master
+    salt-minion
+  ).each do |p|
+    describe service(p) do
+      it { should be_enabled }
+      it { should be_running }
+    end
+  end
+end

test/salt/pillar/develop-py3.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: 'latest'
+  py_ver: 'py3'

test/salt/pillar/salt.sls

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  master:
+    fileserver_backend:
+      - rootfs
+    file_roots:
+      base:
+        - /srv/salt
+    pillar_roots:
+      base:
+        - /srv/pillar
+  minion:
+    master: localhost
+    fileserver_backend:
+      - rootfs
+    file_roots:
+      base:
+        - /srv/salt
+    pillar_roots:
+      base:
+        - /srv/pillar

test/salt/pillar/v2017-7-py2.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: '2017.7'
+  py_ver: 'py2'

test/salt/pillar/v2018-3-py2-forced-version.sls

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: '2018.3'
+  py_ver: 'py2'
+  version: '2018.3.2-5.fc29'

test/salt/pillar/v2018-3-py2.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: '2018.3'
+  py_ver: 'py2'

test/salt/pillar/v2019-2-py2.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: '2019.2'
+  py_ver: 'py2'

test/salt/pillar/v2019-2-py3.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+salt:
+  release: '2019.2'
+  py_ver: 'py3'