overhauled minion template

salt/files/minion

@@ -1,59 +1,66 @@
-# This file managed by Salt, do not edit!!
+# This file managed by Salt, do not edit by hand!!
+#  Based on salt version 0.16.3 default config
 {% set salt = pillar.get('salt', {}) -%}
 {% set minion = salt.get('minion', {}) -%}
-
+{%- macro get_config(configname, default_value) -%}
+{%- if configname in minion -%}
+{{ configname }}: {{ minion[configname] }}
+{%- elif configname in salt -%}
+{{ configname }}: {{ salt[configname] }}
+{%- else -%}
+#{{ configname }}: {{ default_value }}
+{%- endif -%}
+{%- endmacro -%}
 ##### Primary configuration settings #####
 ##########################################
 
 # Per default the minion will automatically include all config files
 # from minion.d/*.conf (minion.d is a directory in the same directory
 # as the main minion config file).
-default_include: {{ minion.get('default_include', 'minion.d/*.conf') }}
+{{ get_config('default_include', 'master.d/*.conf') }}
 
 # Set the location of the salt master server, if the master server cannot be
 # resolved, then the minion will fail to start.
-master: {{ minion.get('master', 'salt') }}
+{{ get_config('master', 'salt') }}
 # Set whether the minion should connect to the master via IPv6
-ipv6: {{ minion.get('ipv6', 'False') }}
+{{ get_config('ipv6', 'False') }}
 
 # Set the number of seconds to wait before attempting to resolve
 # the master hostname if name resolution fails. Defaults to 30 seconds.
 # Set to zero if the minion should shutdown and not retry.
-retry_dns: {{ minion.get('retry_dns', '30') }}
+{{ get_config('retry_dns', '30') }}
 
 # Set the port used by the master reply and authentication server
-master_port: {{ minion.get('master_port', '4506') }}
+{{ get_config('master_port', '4506') }}
 
 # The user to run salt
-user: {{ minion.get('user', 'root') }}
+{{ get_config('user', 'root') }}
 
 # Specify the location of the daemon process ID file
-pidfile: {{ minion.get('pidfile', '/var/run/salt-minion.pid') }}
+{{ get_config('pidfile', '/var/run/salt-minion.pid') }}
 
 # The root directory prepended to these options: pki_dir, cachedir, log_file,
 # sock_dir, pidfile.
-root_dir: {{ minion.get('root_dir', '/') }}
+{{ get_config('root_dir', '/') }}
 
 # The directory to store the pki information in
-pki_dir: {{ minion.get('pki_dir', '/etc/salt/pki/minion') }}
+{{ get_config('pki_dir', '/etc/salt/pki/minion') }}
 
 # Explicitly declare the id for this minion to use, if left commented the id
 # will be the hostname as returned by the python call: socket.getfqdn()
 # Since salt uses detached ids it is possible to run multiple minions on the
 # same machine but with different ids, this can be useful for salt compute
 # clusters.
+{% if 'id' in minion -%}
+id: {{ minion['id'] }}
+{% else -%}
 #id:
-{% if minion['id'] is defined -%}
+{%- endif %}
-id: minion['id']
-{% endif -%}
 
 # Append a domain to a hostname in the event that it does not exist.  This is
 # useful for systems where socket.getfqdn() does not actually result in a
 # FQDN (for instance, Solaris).
-#append_domain:
+{{ get_config('append_domain', '') }}
-{% if minion['append_domain'] is defined -%}
-append_domain: minion['append_domain']
-{% endif -%}
 
 # Custom static grains for this minion can be specified here and used in SLS
 # files just like all other grains. This example sets 4 custom grains, with
@@ -65,29 +72,30 @@ append_domain: minion['append_domain']
 #  deployment: datacenter4
 #  cabinet: 13
 #  cab_u: 14-15
+{{ get_config('grains', '{}') }}
 
 # Where cache data goes
-cachedir: {{ minion.get('cachedir', '/var/cache/salt/minion') }}
+{{ get_config('cachedir', '/var/cache/salt/minion') }}
 
 # Verify and set permissions on configuration directories at startup
-verify_env: {{ minion.get('verify_env', 'True') }}
+{{ get_config('verify_env', 'True') }}
 
 # The minion can locally cache the return data from jobs sent to it, this
 # can be a good way to keep track of jobs the minion has executed
 # (on the minion side). By default this feature is disabled, to enable
 # set cache_jobs to True
-cache_jobs: {{ minion.get('cache_jobs', 'False') }}
+{{ get_config('cache_jobs', 'False') }}
 
 # set the directory used to hold unix sockets
-sock_dir: {{ minion.get('sock_dir', '/var/run/salt/minion') }}
+{{ get_config('sock_dir', '/var/run/salt/minion') }}
 
 # Set the default outputter used by the salt-call command. The default is
 # "nested"
-output: {{ minion.get('output', 'nested') }}
+{{ get_config('output', 'nested') }}
 #
 # By default output is colored, to disable colored output set the color value
 # to False
-color: {{ minion.get('color', 'True') }}
+{{ get_config('color', 'True') }}
 
 # Backup files that are replaced by file.managed and file.recurse under
 # 'cachedir'/file_backups relative to their original location and appended
@@ -100,31 +108,109 @@ color: {{ minion.get('color', 'True') }}
 #     - source: salt://ssh/sshd_config
 #       - backup: minion
 #
-backup_mode: {{ minion.get('backup_mode', 'minion') }}
+{{ get_config('backup_mode', 'minion') }}
 
 # When waiting for a master to accept the minion's public key, salt will
 # continuously attempt to reconnect until successful. This is the time, in
 # seconds, between those reconnection attempts.
-acceptance_wait_time: {{ minion.get('acceptance_wait_time', '10') }}
+{{ get_config('acceptance_wait_time', '10') }}
+
+# If this is nonzero, the time between reconnection attempts will increase by
+# acceptance_wait_time seconds per iteration, up to this maximum. If this is
+# set to zero, the time between reconnection attempts will stay constant.
+{{ get_config('acceptance_wait_time_max', '0') }}
+
+# When the master key changes, the minion will try to re-auth itself to receive
+# the new master key. In larger environments this can cause a SYN flood on the
+# master because all minions try to re-auth immediately. To prevent this and
+# have a minion wait for a random amount of time, use this optional parameter.
+# The wait-time will be a random number of seconds between
+# 0 and the defined value.
+{{ get_config('random_reauth_delay', '60') }}
+
+
+# If you dont have any problems with syn-floods, dont bother with the
+# three recon_* settings described below, just leave the defaults!
+#
+# The ZeroMQ pull-socket that binds to the masters publishing interface tries
+# to reconnect immediately, if the socket is disconnected (for example if
+# the master processes are restarted). In large setups this will have all
+# minions reconnect immediately which might flood the master (the ZeroMQ-default
+# is usually a 100ms delay). To prevent this, these three recon_* settings 
+# can be used.
+#
+# recon_default: the interval in milliseconds that the socket should wait before 
+#                trying to reconnect to the master (100ms = 1 second)
+#
+# recon_max: the maximum time a socket should wait. each interval the time to wait
+#            is calculated by doubling the previous time. if recon_max is reached,
+#            it starts again at recon_default. Short example:
+#
+#            reconnect 1: the socket will wait 'recon_default' milliseconds
+#            reconnect 2: 'recon_default' * 2
+#            reconnect 3: ('recon_default' * 2) * 2
+#            reconnect 4: value from previous interval * 2
+#            reconnect 5: value from previous interval * 2
+#            reconnect x: if value >= recon_max, it starts again with recon_default
+#
+# recon_randomize: generate a random wait time on minion start. The wait time will 
+#                  be a random value between recon_default and recon_default + 
+#                  recon_max. Having all minions reconnect with the same recon_default 
+#                  and recon_max value kind of defeats the purpose of being able to 
+#                  change these settings. If all minions have the same values and your 
+#                  setup is quite large (several thousand minions), they will still 
+#                  flood the master. The desired behaviour is to have timeframe within
+#                  all minions try to reconnect. 
+
+# Example on how to use these settings:
+# The goal: have all minions reconnect within a 60 second timeframe on a disconnect
+#
+# The settings:
+# recon_default: 1000
+# recon_max: 59000
+# recon_randomize: True
+#
+# Each minion will have a randomized reconnect value between 'recon_default'
+# and 'recon_default + recon_max', which in this example means between 1000ms
+# 60000ms (or between 1 and 60 seconds). The generated random-value will be 
+# doubled after each attempt to reconnect. Lets say the generated random 
+# value is 11 seconds (or 11000ms). 
+#
+# reconnect 1: wait 11 seconds
+# reconnect 2: wait 22 seconds
+# reconnect 3: wait 33 seconds
+# reconnect 4: wait 44 seconds
+# reconnect 5: wait 55 seconds
+# reconnect 6: wait time is bigger than 60 seconds (recon_default + recon_max)
+# reconnect 7: wait 11 seconds
+# reconnect 8: wait 22 seconds
+# reconnect 9: wait 33 seconds
+# reconnect x: etc.
+#
+# In a setup with ~6000 thousand hosts these settings would average the reconnects
+# to about 100 per second and all hosts would be reconnected within 60 seconds.
+{{ get_config('recon_default', '100') }}
+{{ get_config('recon_max', '5000') }}
+{{ get_config('recon_randomize', 'False') }}
 
 # The loop_interval sets how long in seconds the minion will wait between
 # evaluating the scheduler and running cleanup tasks. This defaults to a
 # sane 60 seconds, but if the minion scheduler needs to be evaluated more
 # often lower this value
-loop_interval: {{ minion.get('loop_interval', '60') }}
+{{ get_config('loop_interval', '60') }}
 
 # When healing, a dns_check is run. This is to make sure that the originally
 # resolved dns has not changed. If this is something that does not happen in
 # your environment, set this value to False.
-dns_check: {{ minion.get('dns_check', 'True') }}
+{{ get_config('dns_check', 'True') }}
 
 # Windows platforms lack posix IPC and must rely on slower TCP based inter-
 # process communications. Set ipc_mode to 'tcp' on such systems
-ipc_mode: {{ minion.get('ipc_mode', 'ipc') }}
+{{ get_config('ipc_mode', 'ipc') }}
 #
 # Overwrite the default tcp ports used by the minion when in tcp mode
-tcp_pub_port: {{ minion.get('tcp_pub_port', '4510') }}
+{{ get_config('tcp_pub_port', '4510') }}
-tcp_pull_port: {{ minion.get('tcp_pull_port', '4511') }}
+{{ get_config('tcp_pull_port', '4511') }}
 
 # The minion can include configuration from other files. To enable this,
 # pass a list of paths to this option. The paths can be either relative or
@@ -141,7 +227,7 @@ tcp_pull_port: {{ minion.get('tcp_pull_port', '4511') }}
 # include:
 #  - /etc/salt/extra_config
 #  - /etc/roles/webserver
-{% if minion['include'] is defined -%}
+{% if 'include' in minion -%}
     {% if isinstance(minion['include'], list) -%}
         include:
         {% for include in minion['include'] -%}
@@ -150,35 +236,32 @@ tcp_pull_port: {{ minion.get('tcp_pull_port', '4511') }}
     {% else -%}
         include: minion['include']
     {% endif -%}
+{% elif 'include' in salt -%}
+    {% if isinstance(salt['include'], list) -%}
+        include:
+        {% for include in salt['include'] -%}
+            - {{ include }}
+        {% endfor -%}
+    {% else -%}
+        include: salt['include']
+    {% endif -%}
 {% endif -%}
 
 #####   Minion module management     #####
 ##########################################
 # Disable specific modules. This allows the admin to limit the level of
 # access the master has to the minion
-disable_modules: {{ [','.join(
+{{ get_config('disable_modules', '[cmd,test]') }}
-    minion.get('disable_modules', ['cmd', 'test'])
+{{ get_config('disable_returners', '[]') }}
-)] }}
-disable_returners: {{ [ ','.join(
-    minion.get('disable_returners', [])
-) ] }}
 #
 # Modules can be loaded from arbitrary paths. This enables the easy deployment
 # of third party modules. Modules for returners and minions can be loaded.
 # Specify a list of extra directories to search for minion modules and
 # returners. These paths must be fully qualified!
-module_dirs: {{ [ ','.join(
+{{ get_config('module_dirs', '[]') }}
-    minion.get('module_dirs', [])
+{{ get_config('returner_dirs', '[]') }}
-) ] }}
+{{ get_config('states_dirs', '[]') }}
-returner_dirs: {{ [ ','.join(
+{{ get_config('render_dirs', '[]') }}
-    minion.get('returner_dirs', [])
-) ] }}
-states_dirs: {{ [ ','.join(
-    minion.get('states_dirs', [])
-) ] }}
-render_dirs: {{ [ ','.join(
-    minion.get('render_dirs', [])
-) ] }}
 #
 # A module provider can be statically overwritten or extended for the minion
 # via the providers option, in this case the default module will be
@@ -187,9 +270,10 @@ render_dirs: {{ [ ','.join(
 #
 # providers:
 #   pkg: yumpkg5
+{{ get_config('providers', '{}') }}
 #
 # Enable Cython modules searching and loading. (Default: False)
-cython_enable: {{ minion.get('cython_enable', 'False') }}
+{{ get_config('cython_enable', 'False') }}
 #
 
 #####    State Management Settings    #####
@@ -206,47 +290,46 @@ cython_enable: {{ minion.get('cython_enable', 'False') }}
 # json_mako
 # json_wempy
 #
-renderer: {{ minion.get('renderer', 'yaml_jinja') }}
+{{ get_config('renderer', 'yaml_jinja') }}
 #
 # The failhard option tells the minions to stop immediately after the first
 # failure detected in the state execution, defaults to False
-failhard: {{ minion.get('failhard', 'False') }}
+{{ get_config(failhard'', 'False') }}
 #
 # autoload_dynamic_modules Turns on automatic loading of modules found in the
 # environments on the master. This is turned on by default, to turn of
 # autoloading modules when states run set this value to False
-autoload_dynamic_modules: {{ minion.get('autoload_dynamic_modules', 'True') }}
+{{ get_config('autoload_dynamic_modules', 'True') }}
 #
 # clean_dynamic_modules keeps the dynamic modules on the minion in sync with
 # the dynamic modules on the master, this means that if a dynamic module is
 # not on the master it will be deleted from the minion. By default this is
 # enabled and can be disabled by changing this value to False
-clean_dynamic_modules: {{ minion.get('clean_dynamic_modules', 'True') }}
+{{ get_config('clean_dynamic_modules', 'True') }}
 #
 # Normally the minion is not isolated to any single environment on the master
 # when running states, but the environment can be isolated on the minion side
 # by statically setting it. Remember that the recommended way to manage
 # environments is to isolate via the top file.
-environment: {{ minion.get('environment', 'None') }}
+{{ get_config('environment', 'None') }}
 #
 # If using the local file directory, then the state top file name needs to be
 # defined, by default this is top.sls.
-state_top: {{ minion.get('state_top', 'top.sls') }}
+{{ get_config('state_top', 'top.sls') }}
 #
 # Run states when the minion daemon starts. To enable, set startup_states to:
 # 'highstate' -- Execute state.highstate
 # 'sls' -- Read in the sls_list option and execute the named sls files
 # 'top' -- Read top_file option and execute based on that file on the Master
-startup_states: {{ salt.get('startup_states', "''") }}
+{{ get_config('startup_states', "''") }}
 #
 # list of states to run when the minion starts up if startup_states is 'sls'
 #sls_list:
 #  - edit.vim
 #  - hyper
-
+{{ get_config('sls_list', '[]') }}
-#
 # top file to execute if startup_states is 'top'
-top_file: {{ salt.get('top_file', "''") }}
+{{ get_config('', "''") }}
 
 #####     File Directory Settings    #####
 ##########################################
@@ -258,7 +341,7 @@ top_file: {{ salt.get('top_file', "''") }}
 # Set the file client, the client defaults to looking on the master server for
 # files, but can be directed to look at the local file directory setting
 # defined below by setting it to local.
-file_client: {{ minion.get('file_client', 'remote') }}
+{{ get_config('file_client', 'remote') }}
 
 # The file directory works on environments passed to the minion, each environment
 # can have multiple root directories, the subdirectories in the multiple file
@@ -276,21 +359,57 @@ file_client: {{ minion.get('file_client', 'remote') }}
 #     - /srv/salt/prod/states
 #
 # Default:
+{% if 'file_roots' in minion -%}
+file_roots:
+  {% for name, roots in minion['file_roots'].items() -%}
+  {{ name }}:
+    {% for dir in roots -%}
+      - {{ dir }}
+    {% endfor -%}
+  {% endfor -%}
+{% elif 'file_roots' in salt -%}
+file_roots:
+  {% for name, roots in salt['file_roots'].items() -%}
+  {{ name }}:
+    {% for dir in roots -%}
+      - {{ dir }}
+    {% endfor -%}
+  {% endfor -%}
+{% else -%}
 #file_roots:
 #  base:
 #    - /srv/salt
+{%- endif %}
 
 # The hash_type is the hash to use when discovering the hash of a file in
 # the minion directory, the default is md5, but sha1, sha224, sha256, sha384
 # and sha512 are also supported.
-hash_type: {{ minion.get('hash_type', 'md5') }}
+{{ get_config('hash_type', 'md5') }}
 
 # The Salt pillar is searched for locally if file_client is set to local. If
 # this is the case, and pillar data is defined, then the pillar_roots need to
 # also be configured on the minion:
+{% if 'pillar_roots' in master -%}
+pillar_roots:
+  {% for name, roots in master['pillar_roots'].items() -%}
+  {{ name }}:
+    {% for dir in roots -%}
+      - {{ dir }}
+    {% endfor -%}
+  {% endfor -%}
+{% elif 'pillar_roots' in salt -%}
+pillar_roots:
+  {% for name, roots in salt['pillar_roots'].items() -%}
+  {{ name }}:
+    {% for dir in roots -%}
+      - {{ dir }}
+    {% endfor -%}
+  {% endfor -%}
+{% else %}
 #pillar_roots:
 #  base:
 #    - /srv/pillar
+{%- endif %}
 
 ######        Security settings       #####
 ###########################################
@@ -298,35 +417,35 @@ hash_type: {{ minion.get('hash_type', 'md5') }}
 # authentication, this is only intended for highly secure environments or for
 # the situation where your keys end up in a bad state. If you run in open mode
 # you do so at your own risk!
-open_mode: {{ minion.get('open_mode', 'False') }}
+{{ get_config('open_mode', 'False') }}
 
 # Enable permissive access to the salt keys.  This allows you to run the
 # master or minion as root, but have a non-root group be given access to
 # your pki_dir.  To make the access explicit, root must belong to the group
 # you've given access to. This is potentially quite insecure.
-permissive_pki_access: {{ minion.get('permissive_pki_access', 'False') }}
+{{ get_config('permissive_pki_access', 'False') }}
 
 # The state_verbose and state_output settings can be used to change the way
 # state system data is printed to the display. By default all data is printed.
 # The state_verbose setting can be set to True or False, when set to False
 # all data that has a result of True and no changes will be suppressed.
-state_verbose: {{ minion.get('state_verbose', 'True') }}
+{{ get_config('state_verbose', 'True') }}
 #
 # The state_output setting changes if the output is the full multi line
 # output for each changed state if set to 'full', but if set to 'terse'
 # the output will be shortened to a single line.
-state_output: {{ minion.get('state_output', 'full') }}
+{{ get_config('state_output', 'full') }}
 #
 # Fingerprint of the master public key to double verify the master is valid,
 # the master fingerprint can be found by running "salt-key -F master" on the
 # salt master.
-master_finger: {{ salt.get('master_finger', "''") }}
+{{ get_config('master_finger', "''") }}
 
 ######         Thread settings        #####
 ###########################################
 # Disable multiprocessing support, by default when a minion receives a
 # publication a new process is spawned and the command is executed therein.
-multiprocessing: {{ minion.get('multiprocessing', 'True') }}
+{{ get_config('multiprocessing', 'True') }}
 
 #####         Logging settings       #####
 ##########################################
@@ -339,28 +458,28 @@ multiprocessing: {{ minion.get('multiprocessing', 'True') }}
 #log_file: file:///dev/log
 #log_file: udp://loghost:10514
 #
-#log_file: /var/log/salt/minion
+{{ get_config('log_file', '/var/log/salt/minion') }}
-#key_logfile: /var/log/salt/key
+{{ get_config('key_logfile', ' /var/log/salt/key') }}
 #
 # The level of messages to send to the console.
 # One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'.
 # Default: 'warning'
-#log_level: warning
+{{ get_config('log_level', 'warning') }}
 #
 # The level of messages to send to the log file.
 # One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'.
 # Default: 'warning'
-log_level_logfile: {{ minion.get('log_level_logfile', 'warning') }}
+{{ get_config('log_level_logfile', 'warning') }}
 
 # The date and time format used in log messages. Allowed date/time formating
 # can be seen here: http://docs.python.org/library/time.html#time.strftime
-#log_datefmt: '%H:%M:%S'
+{{ get_config('log_datefmt', "'%H:%M:%S'") }}
-#log_datefmt_logfile: '%Y-%m-%d %H:%M:%S'
+{{ get_config('log_datefmt_logfile', "'%Y-%m-%d %H:%M:%S'") }}
 #
 # The format of the console logging messages. Allowed formatting options can
 # be seen here: http://docs.python.org/library/logging.html#logrecord-attributes
-#log_fmt_console: '[%(levelname)-8s] %(message)s'
+{{ get_config('log_fmt_console', "'[%(levelname)-8s] %(message)s'") }}
-#log_fmt_logfile: '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'
+{{ get_config('log_fmt_logfile', "'%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'") }}
 #
 # This can be used to control logging levels more specificically.  This
 # example sets the main salt library at the 'warning' level, but sets
@@ -369,7 +488,7 @@ log_level_logfile: {{ minion.get('log_level_logfile', 'warning') }}
 #     'salt': 'warning',
 #     'salt.modules': 'debug'
 #
-#log_granular_levels: {}
+{{ get_config('log_granular_levels', '{}') }}
 
 ######      Module configuration      #####
 ###########################################
@@ -399,12 +518,10 @@ log_level_logfile: {{ minion.get('log_level_logfile', 'warning') }}
 # (saltutil.update()) behaves.
 #
 # The url for finding and downloading updates. Disabled by default.
-update_url: {{ minion.get('update_url', 'False') }}
+{{ get_config('update_url', 'False') }}
 #
 # The list of services to restart after a successful update. Empty by default.
-update_restart_services: {{ [ ','.join(
+{{ get_config('update_restart_services', '[]') }}
-    minion.get('update_restart_services', [])
-) ] }}
 
 
 ######      Keepalive settings        ######
@@ -418,24 +535,24 @@ update_restart_services: {{ [ ','.join(
 #
 # Overall state of TCP Keepalives, enable (1 or True), disable (0 or False)
 # or leave to the OS defaults (-1), on Linux, typically disabled. Default True, enabled.
-tcp_keepalive: {{ minion.get('tcp_keepalive', 'True') }}
+{{ get_config('tcp_keepalive', 'True') }}
 #
 # How long before the first keepalive should be sent in seconds. Default 300
 # to send the first keepalive after 5 minutes, OS default (-1) is typically 7200 seconds
 # on Linux see /proc/sys/net/ipv4/tcp_keepalive_time.
-tcp_keepalive_idle: {{ minion.get('tcp_keepalive_idle', '300') }}
+{{ get_config('tcp_keepalive_idle', '300') }}
 #
 # How many lost probes are needed to consider the connection lost. Default -1
 # to use OS defaults, typically 9 on Linux, see /proc/sys/net/ipv4/tcp_keepalive_probes.
-tcp_keepalive_cnt: {{ minion.get('tcp_keepalive_cnt', '-1') }}
+{{ get_config('tcp_keepalive_cnt', '-1') }}
 #
 # How often, in seconds, to send keepalives after the first one. Default -1 to
 # use OS defaults, typically 75 seconds on Linux, see
 # /proc/sys/net/ipv4/tcp_keepalive_intvl.
-tcp_keepalive_intvl: {{ minion.get('tcp_keepalive_intvl', '-1') }}
+{{ get_config('tcp_keepalive_intvl', '-1') }}
 
 
 ######      Windows Software settings ######
 ############################################
 # Location of the repository cache file on the master
-# win_repo_cachefile: 'salt://win/repo/winrepo.p'
+{{ get_config('win_repo_cachefile', 'salt://win/repo/winrepo.p') }}