Merge pull request #221 from bradthurber/new-gitfs-functionality

add new gitfs functionality

README.rst

@@ -63,6 +63,11 @@ Install gitfs backend dulwich dependencies. Set ``salt:master:gitfs_provider: du
 
 Install gitfs backend GitPython dependenciess. Set ``salt:master:gitfs_provider: gitpython`` in your pillar.
 
+``salt.gitfs.keys``
+----------------------
+
+Install ssh keys to be used by gitfs
+
 ``salt.gitfs.pygit2``
 ----------------------
 

pillar.example

@@ -115,6 +115,16 @@ salt:
       user: ubuntu
       sudo: True
       priv: /etc/salt/ssh_keys/sshkey.pem
+  gitfs:
+    keys:
+      global:
+        # key and pub end up being the extension used on the key file. values other than key and pub are possible
+        key: |
+          -----BEGIN RSA PRIVATE KEY-----
+          ...........
+          -----END RSA PRIVATE KEY-----
+        pub: |
+          ...........
 
 salt_cloud_certs:
   aws:

salt/files/gitfs_key.jinja

@@ -0,0 +1 @@
+{{ pillar['salt']['gitfs']['keys'][key][type] }}

salt/files/master.d/f_defaults.conf

@@ -699,9 +699,36 @@ fileserver_backend:
 
 # Git File Server Backend Configuration
 #
-# Gitfs can be provided by one of two python modules: GitPython or pygit2. If
-# using pygit2, both libgit2 and git must also be installed.
-{{ get_config('gitfs_provider', 'gitpython') }}
+# Optional parameter used to specify the provider to be used for gitfs. Must
+# be one of the following: pygit2, gitpython, or dulwich. If unset, then each
+# will be tried in that same order, and the first one with a compatible
+# version installed will be the provider that is used.
+{{ get_config('gitfs_provider', 'pygit2') }}
+
+# Along with gitfs_password, is used to authenticate to HTTPS remotes.
+{{ get_config('gitfs_user', 'git') }}
+
+# Along with gitfs_user, is used to authenticate to HTTPS remotes. 
+# This parameter is not required if the repository does not use authentication.
+{{ get_config('gitfs_password', '') }}
+
+# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. 
+# This parameter enables authentication over HTTP. Enable this at your own risk.
+{{ get_config('gitfs_insecure_auth', 'False') }}
+
+# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to
+# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
+# is required for SSH remotes.
+{{ get_config('gitfs_pubkey', '') }}
+
+# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to
+# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
+# is required for SSH remotes.
+{{ get_config('gitfs_privkey', '') }}
+
+# This parameter is optional, required only when the SSH key being used to
+# authenticate is protected by a passphrase.
+{{ get_config('gitfs_passphrase', '') }}
 
 # When using the git fileserver backend at least one git remote needs to be
 # defined. The user running the salt master will need read access to the repo.
@@ -865,8 +892,67 @@ ext_pillar:
 {{ get_config('pillar_source_merging_strategy', 'smart') }}
 
 # Recursively merge lists by aggregating them instead of replacing them.
-{{ get_config('pillar_merge_lists', 'False') }}
+{{ get_config('pillar_merge_lists', False) }}
 
+# Git External Pillar (git_pillar) Configuration Options
+#
+# Specify the provider to be used for git_pillar. Must be either pygit2 or
+# gitpython. If unset, then both will be tried in that same order, and the
+# first one with a compatible version installed will be the provider that
+# is used.
+{{ get_config('git_pillar_provider', 'pygit2') }}
+
+# If the desired branch matches this value, and the environment is omitted
+# from the git_pillar configuration, then the environment for that git_pillar
+# remote will be base.
+{{ get_config('git_pillar_base', 'master') }}
+
+# If the branch is omitted from a git_pillar remote, then this branch will
+# be used instead.
+{{ get_config('git_pillar_branch', 'master') }}
+
+# Environment to use for git_pillar remotes. This is normally derived from 
+# the branch/tag (or from a per-remote env parameter), but if set this will
+# override the process of deriving the env from the branch/tag name. 
+{{ get_config('git_pillar_env', '') }}
+
+# Path relative to the root of the repository where the git_pillar top file
+# and SLS files are located.
+{{ get_config('git_pillar_root', 'pillar') }}
+
+# Specifies whether or not to ignore SSL certificate errors when contacting
+# the remote repository. 
+{{ get_config('git_pillar_ssl_verify', True) }}
+
+# When set to False, if there is an update/checkout lock for a git_pillar
+# remote and the pid written to it is not running on the master, the lock
+# file will be automatically cleared and a new lock will be obtained. 
+{{ get_config('git_pillar_global_lock', False) }}
+
+# Git External Pillar Authentication Options
+#
+# Along with git_pillar_password, is used to authenticate to HTTPS remotes.
+{{ get_config('git_pillar_user', '') }}
+
+# Along with git_pillar_user, is used to authenticate to HTTPS remotes.
+# This parameter is not required if the repository does not use authentication.
+{{ get_config('git_pillar_password', '') }}
+
+# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
+# This parameter enables authentication over HTTP.
+{{ get_config('git_pillar_insecure_auth', False) }}
+
+# Along with git_pillar_privkey (and optionally git_pillar_passphrase),
+# is used to authenticate to SSH remotes.
+{{ get_config('git_pillar_pubkey', '') }}
+
+# Along with git_pillar_pubkey (and optionally git_pillar_passphrase),
+# is used to authenticate to SSH remotes.
+{{ get_config('git_pillar_privkey', '') }}
+
+# This parameter is optional, required only when the SSH key being used
+# to authenticate is protected by a passphrase.
+{{ get_config('git_pillar_passphrase', '') }}
 
 #####          Syndic settings       #####
 ##########################################

salt/files/minion.d/f_defaults.conf

@@ -488,6 +488,33 @@ file_client: local
 {{ file_roots(cfg_salt['file_roots']) }}
 {%- elif formulas|length -%}
 {{ file_roots({'base': ['/srv/salt']}) }}
+{%- else -%}
+#file_roots:
+#  base:
+#    - /srv/salt
+{%- endif %}
+
+
+# File Server Backend
+#
+# Salt supports a modular fileserver backend system, this system allows
+# the salt minion to link directly to third party systems to gather and
+# manage the files available to minions. Multiple backends can be
+# configured and will be searched for the requested file in the order in which
+# they are defined here. The default setting only enables the standard backend
+# "roots" which uses the "file_roots" option.
+#fileserver_backend:
+#  - roots
+#
+# To use multiple backends list them in the order they are searched:
+#fileserver_backend:
+#  - git
+#  - roots
+{% if 'fileserver_backend' in cfg_minion -%}
+fileserver_backend:
+{%- for backend in cfg_minion['fileserver_backend'] %}
+  - {{ backend }}
+{%- endfor -%}
 {%- endif %}
 
 # By default, the Salt fileserver recurses fully into all defined environments
@@ -508,7 +535,40 @@ file_client: local
 
 # gitfs provider
 {{ get_config('gitfs_provider', 'pygit2') }}
-# gitfs remotes
+
+# Along with gitfs_password, is used to authenticate to HTTPS remotes.
+{{ get_config('gitfs_user', 'git') }}
+
+# Along with gitfs_user, is used to authenticate to HTTPS remotes. 
+# This parameter is not required if the repository does not use authentication.
+{{ get_config('gitfs_password', '') }}
+
+# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. 
+# This parameter enables authentication over HTTP. Enable this at your own risk.
+{{ get_config('gitfs_insecure_auth', 'False') }}
+
+# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to
+# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
+# is required for SSH remotes.
+{{ get_config('gitfs_pubkey', '') }}
+
+# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to
+# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
+# is required for SSH remotes.
+{{ get_config('gitfs_privkey', '') }}
+
+# This parameter is optional, required only when the SSH key being used to
+# authenticate is protected by a passphrase.
+{{ get_config('gitfs_passphrase', '') }}
+# When using the git fileserver backend at least one git remote needs to be
+# defined. The user running the salt master will need read access to the repo.
+#
+# The repos will be searched in order to find the file requested by a client
+# and the first repo to have the file will return it.
+# When using the git backend branches and tags are translated into salt
+# environments.
+# Note:  file:// repos will be treated as a remote, so refs you want used must
+# exist in that repo as *local* refs.
 {% if 'gitfs_remotes' in cfg_minion -%}
 gitfs_remotes:
 {%- for remote in cfg_minion['gitfs_remotes'] %}
@@ -526,11 +586,45 @@ gitfs_remotes:
 {%- endif -%}
 {%- endfor -%}
 {%- endif %}
-# verify git ssl errors
+
+# The gitfs_ssl_verify option specifies whether to ignore ssl certificate
+# errors when contacting the gitfs backend. You might want to set this to
+# false if you're using a git backend that uses a self-signed certificate but
+# keep in mind that setting this flag to anything other than the default of True
+# is a security concern, you may want to try using the ssh transport.
 {{ get_config('gitfs_ssl_verify', 'True') }}
-# gitfs root dir
+
+# The gitfs_root option gives the ability to serve files from a subdirectory
+# within the repository. The path is defined relative to the root of the
+# repository and defaults to the repository root.
 {{ get_config('gitfs_root', 'somefolder/otherfolder') }}
 
+# The gitfs_env_whitelist and gitfs_env_blacklist parameters allow for greater
+# control over which branches/tags are exposed as fileserver environments.
+{% if 'gitfs_env_whitelist' in cfg_minion -%}
+gitfs_env_whitelist:
+  {%- for git_env in cfg_minion['gitfs_env_whitelist'] %}
+  - {{ git_env }}
+  {%- endfor -%}
+{% else -%}
+# gitfs_env_whitelist:
+#   - base
+#   - v1.*
+{% endif %}
+
+{% if 'gitfs_env_blacklist' in cfg_minion -%}
+gitfs_env_blacklist:
+  {%- for git_env in cfg_minion['gitfs_env_blacklist'] %}
+  - {{ git_env }}
+  {%- endfor -%}
+{% else -%}
+# gitfs_env_blacklist:
+#   - bug/*
+#   - feature/*
+{% endif %}
+
+#####         Pillar settings        #####
+##########################################
 # The Salt pillar is searched for locally if file_client is set to local. If
 # this is the case, and pillar data is defined, then the pillar_roots need to
 # also be configured on the minion:
@@ -550,8 +644,140 @@ pillar_roots:
     - {{ dir }}
 {%- endfor -%}
 {%- endfor -%}
+{%- else -%}
+#pillar_roots:
+#  base:
+#    - /srv/pillar
 {%- endif %}
 
+{% if 'ext_pillar' in cfg_minion %}
+ext_pillar:
+{%- for pillar in cfg_minion['ext_pillar'] -%}
+  {%- for key in pillar -%}
+    {%- if pillar[key] is string %}
+  - {{ key }}: {{ pillar[key] }}
+    {%- elif pillar[key] is iterable and pillar[key] is not mapping %}
+  - {{ key }}:
+      {%- for parameter in pillar[key] %}
+    - {{ parameter }}
+      {%- endfor -%}
+    {%- elif pillar[key] is mapping and pillar[key] is not string %}
+  - {{ key }}:
+      {%- for parameter in pillar[key] %}
+      {{ parameter }}: {{pillar[key][parameter]}}
+      {%- endfor %}
+    {%- else %}
+# Error in rendering {{ key }}, please read https://docs.saltstack.com/en/latest/topics/development/external_pillars.html#configuration
+    {% endif %}
+  {%- endfor -%}
+{%- endfor %}
+{% elif 'ext_pillar' in cfg_salt %}
+ext_pillar:
+{% for pillar in cfg_salt['ext_pillar'] %}
+  - {{ pillar.items()[0][0] }}: {{ pillar.items()[0][1] }}
+{% endfor %}
+{% else %}
+#ext_pillar:
+#  - hiera: /etc/hiera.yaml
+#  - cmd_yaml: cat /etc/salt/yaml
+{% endif %}
+
+# The ext_pillar_first option allows for external pillar sources to populate
+# before file system pillar. This allows for targeting file system pillar from
+# ext_pillar.
+{{ get_config('ext_pillar_first', 'False') }}
+
+# The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate
+# errors when contacting the pillar gitfs backend. You might want to set this to
+# false if you're using a git backend that uses a self-signed certificate but
+# keep in mind that setting this flag to anything other than the default of True
+# is a security concern, you may want to try using the ssh transport.
+{{ get_config('pillar_gitfs_ssl_verify', 'True') }}
+
+# The pillar_opts option adds the master configuration file data to a dict in
+# the pillar called "master". This is used to set simple configurations in the
+# master config file that can then be used on minions.
+{{ get_config('pillar_opts', 'True') }}
+
+# The pillar_safe_render_error option prevents the master from passing pillar
+# render errors to the minion. This is set on by default because the error could
+# contain templating data which would give that minion information it shouldn't
+# have, like a password! When set true the error message will only show:
+#   Rendering SLS 'my.sls' failed. Please see master log for details.
+{{ get_config('pillar_safe_render_error', 'True') }}
+
+# The pillar_source_merging_strategy option allows you to configure merging strategy
+# between different sources. It accepts four values: recurse, aggregate, overwrite,
+# or smart. Recurse will merge recursively mapping of data. Aggregate instructs
+# aggregation of elements between sources that use the #!yamlex renderer. Overwrite
+# will verwrite elements according the order in which they are processed. This is
+# behavior of the 2014.1 branch and earlier. Smart guesses the best strategy based
+# on the "renderer" setting and is the default value.
+{{ get_config('pillar_source_merging_strategy', 'smart') }}
+
+# Recursively merge lists by aggregating them instead of replacing them.
+{{ get_config('pillar_merge_lists', False) }}
+
+# Git External Pillar (git_pillar) Configuration Options
+#
+# Specify the provider to be used for git_pillar. Must be either pygit2 or
+# gitpython. If unset, then both will be tried in that same order, and the
+# first one with a compatible version installed will be the provider that
+# is used.
+{{ get_config('git_pillar_provider', 'pygit2') }}
+
+# If the desired branch matches this value, and the environment is omitted
+# from the git_pillar configuration, then the environment for that git_pillar
+# remote will be base.
+{{ get_config('git_pillar_base', 'master') }}
+
+# If the branch is omitted from a git_pillar remote, then this branch will
+# be used instead.
+{{ get_config('git_pillar_branch', 'master') }}
+
+# Environment to use for git_pillar remotes. This is normally derived from 
+# the branch/tag (or from a per-remote env parameter), but if set this will
+# override the process of deriving the env from the branch/tag name. 
+{{ get_config('git_pillar_env', '') }}
+
+# Path relative to the root of the repository where the git_pillar top file
+# and SLS files are located.
+{{ get_config('git_pillar_root', 'pillar') }}
+
+# Specifies whether or not to ignore SSL certificate errors when contacting
+# the remote repository. 
+{{ get_config('git_pillar_ssl_verify', True) }}
+
+# When set to False, if there is an update/checkout lock for a git_pillar
+# remote and the pid written to it is not running on the master, the lock
+# file will be automatically cleared and a new lock will be obtained. 
+{{ get_config('git_pillar_global_lock', False) }}
+
+# Git External Pillar Authentication Options
+#
+# Along with git_pillar_password, is used to authenticate to HTTPS remotes.
+{{ get_config('git_pillar_user', '') }}
+
+# Along with git_pillar_user, is used to authenticate to HTTPS remotes.
+# This parameter is not required if the repository does not use authentication.
+{{ get_config('git_pillar_password', '') }}
+
+# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
+# This parameter enables authentication over HTTP.
+{{ get_config('git_pillar_insecure_auth', False) }}
+
+# Along with git_pillar_privkey (and optionally git_pillar_passphrase),
+# is used to authenticate to SSH remotes.
+{{ get_config('git_pillar_pubkey', '') }}
+
+# Along with git_pillar_pubkey (and optionally git_pillar_passphrase),
+# is used to authenticate to SSH remotes.
+{{ get_config('git_pillar_privkey', '') }}
+
+# This parameter is optional, required only when the SSH key being used
+# to authenticate is protected by a passphrase.
+{{ get_config('git_pillar_passphrase', '') }}
+
 
 ######        Security settings       #####
 ###########################################

salt/gitfs/keys.sls

@@ -0,0 +1,20 @@
+{%- from "salt/map.jinja" import salt_settings with context %}
+
+{%- set gitfs_keys=salt['pillar.get']('salt:gitfs:keys') %}
+
+{%- for key, keyvalues in gitfs_keys.items() %}
+{%- for type, keydata in keyvalues.items() %}
+gitfs-key-{{ key }}-{{ type }}:
+  file.managed:
+    - name: {{ salt_settings.config_path }}/pki/gitfs/{{ key }}.{{ type }}
+    - source: salt://salt/files/gitfs_key.jinja
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 600
+    - makedirs: True
+    - defaults:
+        key: {{ key }}
+        type: {{ type }}
+{%- endfor %}
+{%- endfor %}