diff --git a/.gitignore b/.gitignore index 3a6cc82..4aa68cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ +*.egg +*.egg-info/ +*.py[cod] .*.sw? +.env +.pytest_cache/ +.venv/ /.idea/ +__pycache__/ +Dockerfile.*_* +ignore/ +tmp/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9894711 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +env: + matrix: + - OS_ID: centos_master_2017.7.2 + - OS_ID: debian_master_2017.7.2 + - OS_ID: opensuse_master_2017.7.2 + - OS_ID: ubuntu_master_2016.11.3 + - OS_ID: ubuntu_master_2017.7.2 + +sudo: required + +language: python + +services: +- docker + +before_install: +- pip install Jinja2 +- python ${TRAVIS_BUILD_DIR}/tools/filltmpl.py nginx ${OS_ID} + +install: +- docker build --force-rm -t "nginx:salt-testing-${OS_ID}" -f "Dockerfile.${OS_ID}" . + +script: +- ./tools/run-tests.sh nginx ${OS_ID} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..068031f --- /dev/null +++ b/Makefile @@ -0,0 +1,80 @@ +FORMULA_NAME = "nginx" +PWD = $(shell pwd) + +# --------------------------------------------------------------- +define render_dockerfile + python $(PWD)/tools/filltmpl.py $(FORMULA_NAME) $(1) +endef + +define docker_build + docker build --force-rm -t $(FORMULA_NAME):salt-testing-$(1) -f Dockerfile.$(1) . +endef + +define docker_run_local + docker run --rm -v $(PWD):/opt/$(FORMULA_NAME)-formula --env=STAGE=TEST -h salt-testing-$(1) --name salt-testing-$(1) -it $(FORMULA_NAME):salt-testing-$(1) /bin/bash +endef + +define run_tests + ./tools/run-tests.sh $(FORMULA_NAME) $(1) +endef + +# --- convenience functions ------------------------------------- +define build_thing + $(call render_dockerfile,$(1)) && $(call docker_build,$(1)) +endef + +define run_local_tests + $(call build_thing,$(1)) && $(call run_tests,$(1)) +endef + +define run_local + $(call build_thing,$(1)) && $(call docker_run_local,$(1)) +endef + +# --------------------------------------------------------------- +setup: + pip install Jinja2 + +clean: + find . -name '*.pyc' -exec rm '{}' ';' + rm -rf Dockerfile.* + # delete pytest caches... + # rm -rf tests/pytests/*/.pytest_cache + # rm -rf tests/pytests/*/__pycache__ + rm -rf tests/pytests/apply-all-tests/.pytest_cache + rm -rf tests/pytests/apply-all-tests/__pycache__ + +# --- centos_master_2017.7.2 ------------------------------------ +test-centos_master_2017.7.2: clean + $(call run_local_tests,centos_master_2017.7.2) + +local-centos_master_2017.7.2: clean + $(call run_local,centos_master_2017.7.2) + +# --- debian_master_2017.7.2 ------------------------------------ +test-debian_master_2017.7.2: clean + $(call run_local_tests,debian_master_2017.7.2) + +local-debian_master_2017.7.2: clean + $(call run_local,debian_master_2017.7.2) + +# --- opensuse_master_2017.7.2 ------------------------------------ +test-opensuse_master_2017.7.2: clean + $(call run_local_tests,opensuse_master_2017.7.2) + +local-opensuse_master_2017.7.2: clean + $(call run_local,opensuse_master_2017.7.2) + +# --- ubuntu_master_2016.11.3 ------------------------------------ +test-ubuntu_master_2016.11.3: clean + $(call run_local_tests,ubuntu_master_2016.11.3) + +local-ubuntu_master_2016.11.3: clean + $(call run_local,ubuntu_master_2016.11.3) + +# --- ubuntu_master_2017.7.2 ------------------------------------ +test-ubuntu_master_2017.7.2: clean + $(call run_local_tests,ubuntu_master_2017.7.2) + +local-ubuntu_master_2017.7.2: clean + $(call run_local,ubuntu_master_2017.7.2) diff --git a/README.rst b/README.rst index b363633..b171207 100644 --- a/README.rst +++ b/README.rst @@ -49,10 +49,10 @@ Installs nginx via the source files. ``nginx.users`` --------------- -Installs apache utils, and configures nginx users specified in the pillar. -This requires `basicauth `_ -from `salt-contrib `_ (either add it to your salt or ship -this single file in your `_modules` directory see `Dynamic Module Distribution +Installs apache utils, and configures nginx users specified in the pillar. +This requires `basicauth `_ +from `salt-contrib `_ (either add it to your salt or ship +this single file in your `_modules` directory see `Dynamic Module Distribution `_ Next-generation, alternate approach @@ -73,10 +73,20 @@ Meta-state for inclusion of all ng states. **Note:** nginx.ng requires the merge parameter of salt.modules.pillar.get(), first available in the Helium release. -``nginx.ng.install`` +``nginx.ng.pkg`` -------------------- -Installs the nginx package. +Installs nginx from package, from the distribution repositories, the official nginx repo or the ppa from Launchpad. + +``nginx.ng.src`` +-------------------- + +Builds and installs nginx from source. + +``nginx.ng.certificates`` +------------------- + +Manages the deployment of nginx certificates. ``nginx.ng.config`` ------------------- @@ -98,3 +108,57 @@ and does not bind them to service calls. ------------------- Manages nginx virtual hosts files and binds them to service calls. + +``nginx.ng.passenger`` +---------------------- + +Installs and configures Phusion Passenger module for nginx. You need to enable +the upstream phusion passenger repository with `install_from_phusionpassenger: true`. +Nginx will also be installed from that repository, as it needs to be modified to +allow the passenger module to work. + + + +Running Tests +============= + +This test runner was implemented using the formula-test-harness_ project. + +Tests will be run on the following base images: + +* ``simplyadrian/allsalt:centos_master_2017.7.2`` +* ``simplyadrian/allsalt:debian_master_2017.7.2`` +* ``simplyadrian/allsalt:opensuse_master_2017.7.2`` +* ``simplyadrian/allsalt:ubuntu_master_2016.11.3`` +* ``simplyadrian/allsalt:ubuntu_master_2017.7.2`` + +Local Setup +----------- + +.. code-block:: shell + + pip install -U virtualenv + virtualenv .venv + source .venv/bin/activate + make setup + +Run tests +--------- + +* ``make test-centos_master_2017.7.2`` +* ``make test-debian_master_2017.7.2`` +* ``make test-opensuse_master_2017.7.2`` +* ``make test-ubuntu_master_2016.11.3`` +* ``make test-ubuntu_master_2017.7.2`` + +Run Containers +-------------- + +* ``make local-centos_master_2017.7.2`` +* ``make local-debian_master_2017.7.2`` +* ``make local-opensuse_master_2017.7.2`` +* ``make local-ubuntu_master_2016.11.3`` +* ``make local-ubuntu_master_2017.7.2`` + + +.. _formula-test-harness: https://github.com/intuitivetechnologygroup/formula-test-harness diff --git a/nginx/common.sls b/nginx/common.sls index 93d0972..d26ad81 100644 --- a/nginx/common.sls +++ b/nginx/common.sls @@ -1,7 +1,7 @@ {% from "nginx/map.jinja" import nginx as nginx_map with context %} {% set nginx = pillar.get('nginx', {}) -%} -{% set home = nginx.get('home', '/var/www') -%} -{% set conf_dir = nginx.get('conf_dir', '/etc/nginx') -%} +{% set home = nginx.get('home', nginx_map.home) -%} +{% set conf_dir = nginx.get('conf_dir', nginx_map.conf_dir) -%} {% set conf_template = nginx.get('conf_template', 'salt://nginx/templates/config.jinja') -%} {{ home }}: diff --git a/nginx/map.jinja b/nginx/map.jinja index b467106..67f93ee 100644 --- a/nginx/map.jinja +++ b/nginx/map.jinja @@ -41,4 +41,25 @@ 'install_prefix': '/usr/local/nginx', 'make_flags': '-j2' }, + 'Suse': { + 'apache_utils': 'apache2-utils', + 'group_action': 'pkg.installed', + 'group_pkg': 'patterns-devel-base-devel_rpm_build', + 'libpcre_dev': 'pcre-devel', + 'libssl_dev': 'openssl-devel', + 'pid_path': '/run/nginx.pid', + 'package': 'nginx', + 'default_user': 'nginx', + 'default_group': 'nginx', + 'disable_before_rename': True, + 'old_init_disable': 'chkconfig --del nginx', + 'use_upstart': False, + 'use_sysvinit': False, + 'home': '/srv/www', + 'conf_dir': '/etc/nginx', + 'log_dir': '/var/log/nginx', + 'sbin_dir': '/usr/sbin', + 'install_prefix': '/usr/local/nginx', + 'make_flags': '-j2' + }, }, grain='os_family', merge=salt['pillar.get']('nginx:lookup'), default='Debian') %} diff --git a/nginx/ng/certificates.sls b/nginx/ng/certificates.sls index db3e05a..27e1396 100644 --- a/nginx/ng/certificates.sls +++ b/nginx/ng/certificates.sls @@ -1,23 +1,60 @@ +{% from 'nginx/ng/map.jinja' import nginx with context %} + include: - nginx.ng.service +{% set certificates_path = salt['pillar.get']('nginx:ng:certificates_path', '/etc/nginx/ssl') %} + +{%- for dh_param, value in salt['pillar.get']('nginx:ng:dh_param', {}).items() %} +{%- if value is string %} +create_nginx_dhparam_{{ dh_param }}_key: + file.managed: + - name: {{ certificates_path }}/{{ dh_param }} + - contents_pillar: nginx:ng:dh_param:{{ dh_param }} + - makedirs: True + - watch_in: + - service: nginx_service +{%- else %} +generate_nginx_dhparam_{{ dh_param }}_key: + pkg.installed: + - name: {{ nginx.lookup.openssl_package }} + file.directory: + - name: {{ certificates_path }} + - makedirs: True + cmd.run: + - name: openssl dhparam -out {{ dh_param }} {{ value.get('keysize', 2048) }} + - cwd: {{ certificates_path }} + - creates: {{ certificates_path }}/{{ dh_param }} + - watch_in: + - service: nginx_service +{%- endif %} +{%- endfor %} + {%- for domain in salt['pillar.get']('nginx:ng:certificates', {}).keys() %} nginx_{{ domain }}_ssl_certificate: file.managed: - - name: /etc/nginx/ssl/{{ domain }}.crt + - name: {{ certificates_path }}/{{ domain }}.crt - makedirs: True +{% if salt['pillar.get']("nginx:ng:certificates:{}:public_cert_pillar".format(domain)) %} + - contents_pillar: {{salt['pillar.get']('nginx:ng:certificates:{}:public_cert_pillar'.format(domain))}} +{% else %} - contents_pillar: nginx:ng:certificates:{{ domain }}:public_cert +{% endif %} - watch_in: - service: nginx_service -{% if salt['pillar.get']("nginx:ng:certificates:{}:private_key".format(domain)) %} +{% if salt['pillar.get']("nginx:ng:certificates:{}:private_key".format(domain)) or salt['pillar.get']("nginx:ng:certificates:{}:private_key_pillar".format(domain))%} nginx_{{ domain }}_ssl_key: file.managed: - - name: /etc/nginx/ssl/{{ domain }}.key + - name: {{ certificates_path }}/{{ domain }}.key - mode: 600 - makedirs: True +{% if salt['pillar.get']("nginx:ng:certificates:{}:private_key_pillar".format(domain)) %} + - contents_pillar: {{salt['pillar.get']('nginx:ng:certificates:{}:private_key_pillar'.format(domain))}} +{% else %} - contents_pillar: nginx:ng:certificates:{{ domain }}:private_key +{% endif %} - watch_in: - service: nginx_service {% endif %} diff --git a/nginx/ng/config.sls b/nginx/ng/config.sls index 71dec31..1fc5dfa 100644 --- a/nginx/ng/config.sls +++ b/nginx/ng/config.sls @@ -12,11 +12,18 @@ nginx_log_dir: - group: {{ nginx.server.config.user }} {% endif %} +{% if 'source_path' in nginx.server.config %} +{% set source_path = nginx.server.config.source_path %} +{% else %} +{% set source_path = 'salt://nginx/ng/files/nginx.conf' %} +{% endif %} nginx_config: file.managed: {{ sls_block(nginx.server.opts) }} - name: {{ nginx.lookup.conf_file }} - - source: salt://nginx/ng/files/nginx.conf + - source: {{ source_path }} - template: jinja +{% if 'source_path' not in nginx.server.config %} - context: config: {{ nginx.server.config|json(sort_keys=False) }} +{% endif %} diff --git a/nginx/ng/files/nginx.conf b/nginx/ng/files/nginx.conf index 05d8f01..91efad6 100644 --- a/nginx/ng/files/nginx.conf +++ b/nginx/ng/files/nginx.conf @@ -32,6 +32,14 @@ # # This file is managed by Salt. +{% if 'load_module' in config.keys() %} +{{ nginx_block(config.pop('load_module'), 'load_module') }} +{%- endif -%} + +{% if 'include' in config.keys() %} +{{ nginx_block(config.pop('include'), 'include') }} +{%- endif -%} + {% for key, value in config.items() %} {{ nginx_block(value, key) }} {%- endfor -%} diff --git a/nginx/ng/init.sls b/nginx/ng/init.sls index 3af5427..411aeec 100644 --- a/nginx/ng/init.sls +++ b/nginx/ng/init.sls @@ -7,6 +7,9 @@ include: - nginx.ng.config - nginx.ng.service + {% if nginx.snippets is defined %} + - nginx.ng.snippets + {% endif %} - nginx.ng.servers - nginx.ng.certificates diff --git a/nginx/ng/map.jinja b/nginx/ng/map.jinja index 0b197bc..e375acd 100644 --- a/nginx/ng/map.jinja +++ b/nginx/ng/map.jinja @@ -8,39 +8,56 @@ 'lookup': salt['grains.filter_by']({ 'Debian': { 'package': 'nginx', + 'passenger_package': 'passenger', + 'passenger_config_file': '/etc/nginx/conf.d/passenger.conf', 'service': 'nginx', 'webuser': 'www-data', 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/sites-available', 'server_enabled': '/etc/nginx/sites-enabled', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': True, 'pid_file': '/run/nginx.pid', + 'openssl_package': 'openssl', }, 'CentOS': { 'package': 'nginx', + 'passenger_package': 'passenger', + 'passenger_config_file': '/etc/nginx/conf.d/passenger.conf', 'service': 'nginx', 'webuser': 'nginx', 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/conf.d', 'server_enabled': '/etc/nginx/conf.d', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': False, 'pid_file': '/run/nginx.pid', 'rh_os_releasever': '$releasever', 'gpg_check': False, 'gpg_key': 'http://nginx.org/keys/nginx_signing.key', + 'openssl_package': 'openssl', }, 'RedHat': { 'package': 'nginx', + 'passenger_package': 'passenger', + 'passenger_config_file': '/etc/nginx/conf.d/passenger.conf', 'service': 'nginx', 'webuser': 'nginx', 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/conf.d', 'server_enabled': '/etc/nginx/conf.d', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': False, 'pid_file': '/run/nginx.pid', 'rh_os_releasever': '$releasever', 'gpg_check': False, 'gpg_key': 'http://nginx.org/keys/nginx_signing.key', + 'passenger': { + 'passenger_root': '/usr/share/ruby/vendor_ruby/phusion_passenger/locations.ini', + 'passenger_instance_registry_dir': ' /var/run/passenger-instreg', + 'passenger_ruby': '/usr/bin/ruby', + }, + 'openssl_package': 'openssl', }, 'Suse': { 'package': 'nginx', @@ -49,10 +66,12 @@ 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/conf.d', 'server_enabled': '/etc/nginx/conf.d', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': False, 'pid_file': '/run/nginx.pid', 'gpg_check': True, - 'gpg_key': 'http://download.opensuse.org/repositories/server:/http/openSUSE_13.2/repodata/repomd.xml.key' + 'gpg_key': 'http://download.opensuse.org/repositories/server:/http/openSUSE_13.2/repodata/repomd.xml.key', + 'openssl_package': 'openssl', }, 'Arch': { 'package': 'nginx', @@ -61,7 +80,9 @@ 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/sites-available', 'server_enabled': '/etc/nginx/sites-enabled', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': True, + 'openssl_package': 'openssl', }, 'Gentoo': { 'package': 'www-servers/nginx', @@ -70,12 +91,26 @@ 'conf_file': '/etc/nginx/nginx.conf', 'server_available': '/etc/nginx/sites-available', 'server_enabled': '/etc/nginx/sites-enabled', + 'snippets_dir': '/etc/nginx/snippets', 'server_use_symlink': True, + 'openssl_package': 'dev-libs/openssl', + }, + 'FreeBSD': { + 'package': 'nginx', + 'passenger_package': 'passenger', + 'service': 'nginx', + 'webuser': 'www', + 'conf_file': '/usr/local/etc/nginx/nginx.conf', + 'server_available': '/usr/local/etc/nginx/sites-available', + 'server_enabled': '/usr/local/etc/nginx/sites-enabled', + 'server_use_symlink': True, + 'pid_file': '/var/run/nginx.pid', }, }, default='Debian' ), 'install_from_source': False, 'install_from_ppa': False, 'install_from_repo': False, + 'install_from_phusionpassenger': False, 'ppa_version': 'stable', 'source_version': '1.10.0', 'source_hash': '8ed647c3dd65bc4ced03b0e0f6bf9e633eff6b01bac772bcf97077d58bc2be4d', @@ -108,9 +143,9 @@ 'gzip': 'off', 'gzip_disable': '"msie6"', 'include': [ - '/etc/nginx/mime.types', - '/etc/nginx/conf.d/*.conf', - '/etc/nginx/sites-enabled/*', + 'mime.types', + 'conf.d/*.conf', + 'sites-enabled/*', ], }, }, @@ -119,12 +154,18 @@ 'disabled_postfix': '.disabled', 'symlink_opts': {}, 'rename_opts': {}, - 'managed_opts': {}, + 'managed_opts': { + 'makedirs': True, + }, 'dir_opts': { 'makedirs': True, }, 'managed': {}, }, + 'passenger': { + 'passenger_root': '/usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini', + 'passenger_ruby': '/usr/bin/ruby', + }, }, merge=True) %} {% if 'user' not in nginx.server.config %} @@ -139,3 +180,17 @@ })%} {% endif %} +{% if salt['grains.get']('os_family') == 'RedHat' %} +{% do nginx.passenger.update({ + 'passenger_root': '/usr/share/ruby/vendor_ruby/phusion_passenger/locations.ini', + 'passenger_instance_registry_dir': '/var/run/passenger-instreg', +})%} + {% if salt['grains.get']('osfinger') == 'CentOS-6' %} + {% do nginx.server.config.update({ + 'pid': '/var/run/nginx.pid', + })%} + {% do nginx.passenger.update({ + 'passenger_root': '/usr/lib/ruby/1.8/phusion_passenger/locations.ini', + })%} + {% endif %} +{% endif %} diff --git a/nginx/ng/passenger.sls b/nginx/ng/passenger.sls new file mode 100644 index 0000000..220056b --- /dev/null +++ b/nginx/ng/passenger.sls @@ -0,0 +1,41 @@ +# nginx.ng.passenger +# +# Manages installation of passenger from repo. +# Requires install_from_phusionpassenger = True + +{% from 'nginx/ng/map.jinja' import nginx, sls_block with context %} + +{% if salt['grains.get']('os_family') in ['Debian', 'RedHat'] %} +include: + - nginx.ng.pkg + - nginx.ng.service + +passenger_install: + pkg.installed: + - name: {{ nginx.lookup.passenger_package }} + - require: + - pkg: nginx_install + - require_in: + - service: nginx_service + +/etc/nginx/passenger.conf: + file.absent: + - require: + - pkg: passenger_install + +passenger_config: + file.managed: + {{ sls_block(nginx.server.opts) }} + - name: {{ nginx.lookup.passenger_config_file }} + - source: salt://nginx/ng/files/nginx.conf + - template: jinja + - context: + config: {{ nginx.passenger|json() }} + - watch_in: + - service: nginx_service + - require_in: + - service: nginx_service + - require: + - file: /etc/nginx/passenger.conf + - pkg: passenger_install +{% endif %} diff --git a/nginx/ng/pkg.sls b/nginx/ng/pkg.sls index 35baaf1..b68cd1b 100644 --- a/nginx/ng/pkg.sls +++ b/nginx/ng/pkg.sls @@ -3,30 +3,57 @@ # Manages installation of nginx from pkg. {% from 'nginx/ng/map.jinja' import nginx, sls_block with context %} +{%- if nginx.install_from_repo %} + {% set from_official = true %} + {% set from_ppa = false %} + {% set from_phusionpassenger = false %} +{% elif nginx.install_from_ppa %} + {% set from_official = false %} + {% set from_ppa = true %} + {% set from_phusionpassenger = false %} +{% elif nginx.install_from_phusionpassenger %} + {% set from_official = false %} + {% set from_ppa = false %} + {% set from_phusionpassenger = true %} +{% else %} + {% set from_official = false %} + {% set from_ppa = false %} + {% set from_phusionpassenger = false %} +{%- endif %} nginx_install: pkg.installed: {{ sls_block(nginx.package.opts) }} + {% if nginx.lookup.package is iterable and nginx.lookup.package is not string %} + - pkgs: + {% for pkg in nginx.lookup.package %} + - {{ pkg }} + {% endfor %} + {% else %} - name: {{ nginx.lookup.package }} + {% endif %} {% if salt['grains.get']('os_family') == 'Debian' %} - {%- if nginx.install_from_repo %} -nginx-official-repo: +nginx_official_repo: pkgrepo: + {%- if from_official %} - managed + {%- else %} + - absent + {%- endif %} - humanname: nginx apt repo - name: deb http://nginx.org/packages/{{ grains['os'].lower() }}/ {{ grains['oscodename'] }} nginx - file: /etc/apt/sources.list.d/nginx-official-{{ grains['oscodename'] }}.list - keyid: ABF5BD827BD9BF62 - keyserver: keyserver.ubuntu.com - require_in: - - pkg: nginx + - pkg: nginx_install - watch_in: - - pkg: nginx - {%- else %} + - pkg: nginx_install + nginx_ppa_repo: pkgrepo: - {%- if nginx.install_from_ppa %} + {%- if from_ppa %} - managed {%- else %} - absent @@ -42,13 +69,29 @@ nginx_ppa_repo: - pkg: nginx_install - watch_in: - pkg: nginx_install - {%- endif %} + +nginx_phusionpassenger_repo: + pkgrepo: + {%- if from_phusionpassenger %} + - managed + {%- else %} + - absent + {%- endif %} + - humanname: nginx phusionpassenger repo + - name: deb https://oss-binaries.phusionpassenger.com/apt/passenger {{ grains['oscodename'] }} main + - file: /etc/apt/sources.list.d/nginx-phusionpassenger-{{ grains['oscodename'] }}.list + - keyid: 561F9B9CAC40B2F7 + - keyserver: keyserver.ubuntu.com + - require_in: + - pkg: nginx_install + - watch_in: + - pkg: nginx_install {% endif %} -{% if salt['grains.get']('os_family') == 'Suse' %} +{% if salt['grains.get']('os_family') == 'Suse' or salt['grains.get']('os') == 'SUSE' %} nginx_zypp_repo: pkgrepo: - {%- if nginx.install_from_repo %} + {%- if from_official %} - managed {%- else %} - absent @@ -68,11 +111,12 @@ nginx_zypp_repo: {% if salt['grains.get']('os_family') == 'RedHat' %} nginx_yum_repo: - {%- if nginx.install_from_repo %} - pkgrepo.managed: - {%- else %} - pkgrepo.absent: - {%- endif %} + pkgrepo: + {%- if from_official %} + - managed + {%- else %} + - absent + {%- endif %} - name: nginx - humanname: nginx repo {%- if salt['grains.get']('os') == 'CentOS' %} @@ -87,4 +131,25 @@ nginx_yum_repo: - pkg: nginx_install - watch_in: - pkg: nginx_install + +nginx_phusionpassenger_yum_repo: + pkgrepo: + {%- if from_phusionpassenger %} + - managed + {%- else %} + - absent + {%- endif %} + - name: passenger + - humanname: nginx phusionpassenger repo + - baseurl: 'https://oss-binaries.phusionpassenger.com/yum/passenger/el/$releasever/$basearch' + - repo_gpgcheck: 1 + - gpgcheck: 0 + - gpgkey: 'https://packagecloud.io/gpg.key' + - enabled: True + - sslverify: 1 + - sslcacert: /etc/pki/tls/certs/ca-bundle.crt + - require_in: + - pkg: nginx_install + - watch_in: + - pkg: nginx_install {% endif %} diff --git a/nginx/ng/servers_config.sls b/nginx/ng/servers_config.sls index e2e2685..ae2b59f 100644 --- a/nginx/ng/servers_config.sls +++ b/nginx/ng/servers_config.sls @@ -37,7 +37,7 @@ {%- endmacro %} # Creates the sls block that manages symlinking / renaming servers -{% macro manage_status(server, state) -%} +{% macro manage_status(server, state, deleted) -%} {%- set anti_state = {True:False, False:True}.get(state) -%} {% if state == True %} {%- if nginx.lookup.server_use_symlink %} @@ -46,20 +46,30 @@ - name: {{ server_path(server, state) }} - target: {{ server_path(server, anti_state) }} {%- else %} + {%- if deleted == True %} + file.absent: + - name: {{ server_path(server, state) }} + {%- else %} file.rename: {{ sls_block(nginx.servers.rename_opts) }} - name: {{ server_path(server, state) }} - source: {{ server_path(server, anti_state) }} + {%- endif %} {%- endif %} {%- elif state == False %} {%- if nginx.lookup.server_use_symlink %} file.absent: - name: {{ server_path(server, anti_state) }} {%- else %} + {%- if deleted == True %} + file.absent: + - name: {{ server_path(server, state) }} + {%- else %} file.rename: {{ sls_block(nginx.servers.rename_opts) }} - name: {{ server_path(server, state) }} - source: {{ server_path(server, anti_state) }} + {%- endif %} {%- endif -%} {%- endif -%} {%- endmacro %} @@ -84,32 +94,57 @@ nginx_server_available_dir: # Managed enabled/disabled state for servers {% for server, settings in nginx.servers.managed.items() %} -{% if settings.config != None %} {% set conf_state_id = 'server_conf_' ~ loop.index0 %} +{% if 'deleted' in settings and settings.deleted %} +{{ conf_state_id }}: + file.absent: + - name: {{ server_curpath(server) }} +{% else %} +{% if settings.config != None and settings.enabled == True %} +{% if 'source_path' in settings.config %} +{% set source_path = settings.config.source_path %} +{% else %} +{% set source_path = 'salt://nginx/ng/files/server.conf' %} +{% endif %} {{ conf_state_id }}: file.managed: {{ sls_block(nginx.servers.managed_opts) }} - name: {{ server_curpath(server) }} - - source: salt://nginx/ng/files/server.conf + - source: {{ source_path }} - template: jinja + - require_in: + - service: nginx_service +{% if 'source_path' not in settings.config %} - context: config: {{ settings.config|json(sort_keys=False) }} +{% endif %} {% if 'overwrite' in settings and settings.overwrite == False %} - unless: - test -e {{ server_curpath(server) }} {% endif %} {% do server_states.append(conf_state_id) %} {% endif %} +{% endif %} {% if settings.enabled != None %} {% set status_state_id = 'server_state_' ~ loop.index0 %} +{%- set enabled_dir = path_join(server, nginx.servers.managed.get(server).get('enabled_dir', nginx.lookup.server_enabled)) -%} +{%- set available_dir = path_join(server, nginx.servers.managed.get(server).get('available_dir', nginx.lookup.server_available)) -%} +{%- if enabled_dir != available_dir %} {{ status_state_id }}: -{{ manage_status(server, settings.enabled) }} -{% if settings.config != None %} +{% if 'deleted' in settings and settings.deleted %} +{{ manage_status(server, False, True) }} +{% else %} +{{ manage_status(server, settings.enabled, False) }} +{% endif %} +{% if settings.config != None and settings.enabled == True %} - require: - file: {{ conf_state_id }} {% endif %} +{% if 'deleted' not in settings or ( 'deleted' in settings and settings.deleted == False ) %} {% do server_states.append(status_state_id) %} {% endif %} +{%- endif %} {# enabled != available_dir #} +{% endif %} {% endfor %} diff --git a/nginx/ng/service.sls b/nginx/ng/service.sls index 2cc51c6..59fe80f 100644 --- a/nginx/ng/service.sls +++ b/nginx/ng/service.sls @@ -17,8 +17,8 @@ nginx_systemd_service_file: file.managed: - name: /lib/systemd/system/nginx.service - source: salt://nginx/ng/files/nginx.service -{% endif %} - +{% endif %} + nginx_service: service.{{ service_function }}: {{ sls_block(nginx.service.opts) }} diff --git a/nginx/ng/snippets.sls b/nginx/ng/snippets.sls new file mode 100644 index 0000000..8635f2d --- /dev/null +++ b/nginx/ng/snippets.sls @@ -0,0 +1,20 @@ +# nginx.ng.snippet +# +# Manages creation of snippets + +{% from 'nginx/ng/map.jinja' import nginx, sls_block with context %} + +nginx_snippets_dir: + file.directory: + {{ sls_block(nginx.servers.dir_opts) }} + - name: {{ nginx.lookup.snippets_dir }} + +{% for snippet, config in nginx.snippets.items() %} +nginx_snippet_{{ snippet }}: + file.managed: + - name: {{ nginx.lookup.snippets_dir }}/{{ snippet }}.conf + - source: salt://nginx/ng/files/server.conf + - template: jinja + - context: + config: {{ config|json() }} +{% endfor %} diff --git a/nginx/openresty.sls b/nginx/openresty.sls index fc51a17..23764df 100644 --- a/nginx/openresty.sls +++ b/nginx/openresty.sls @@ -21,10 +21,10 @@ get-openresty: install_openresty: cmd.wait: - cwd: {{ home }}/ngx_openresty-{{ openresty_version }} - - names: + - names: - ./configure --with-luajit \ --with-http_drizzle_module \ - --with-http_postgres_module \ + --with-http_postgres_module \ --with-http_iconv_module - make && make install - watch: diff --git a/nginx/package.sls b/nginx/package.sls index 3a765e5..3229ae2 100644 --- a/nginx/package.sls +++ b/nginx/package.sls @@ -102,7 +102,7 @@ nginx: - require: - pkg: nginx - file: nginx-old-init - - module: nginx-old-init + - module: nginx-old-init {% endif %} service.running: - enable: True diff --git a/nginx/source.sls b/nginx/source.sls index 3d13f48..5f27470 100644 --- a/nginx/source.sls +++ b/nginx/source.sls @@ -170,7 +170,11 @@ nginx: - cwd: {{ nginx_source }} - names: - ( + {%- if nginx.get('debug_symbols', false) %} + CFLAGS="-g -O0" ./configure --conf-path={{ conf_dir }}/nginx.conf + {%- else %} ./configure --conf-path={{ conf_dir }}/nginx.conf + {%- endif %} --sbin-path={{ sbin_dir }}/nginx --user={{ nginx_map.default_user }} --group={{ nginx_map.default_group }} @@ -216,7 +220,7 @@ nginx: {% if use_sysvinit %} - watch_in: {% set logger_types = ('access', 'error') %} - {% for log_type in logger_types %} + {% for log_type in logger_types %} - service: nginx-logger-{{ log_type }} {% endfor %} {% endif %} diff --git a/nginx/templates/config.jinja b/nginx/templates/config.jinja index 4c36c3d..3d0a94f 100644 --- a/nginx/templates/config.jinja +++ b/nginx/templates/config.jinja @@ -13,7 +13,7 @@ worker_rlimit_nofile {{ worker_rlimit_nofile }}; {% set error_log_level = nginx.get('error_log',{}).get('level', 'warn') -%} error_log {{ ' '.join([error_log_location, error_log_level]) }}; pid {{ nginx.get('pid', '/var/run/nginx.pid') }}; -{% if salt['test.provider']('service') != 'systemd' -%} +{% if not 'systemd' in salt['test.provider']('service') -%} daemon {{ nginx.get('daemon', 'on') }}; {%- endif %} diff --git a/nginx/templates/upstart.jinja b/nginx/templates/upstart.jinja index a67fdd3..64bab02 100644 --- a/nginx/templates/upstart.jinja +++ b/nginx/templates/upstart.jinja @@ -1,23 +1,23 @@ # nginx - + description "nginx http daemon" author "George Shammas " - + start on (runlevel [345] and started network) stop on (runlevel [!345] or stopping network) - + env DAEMON=/usr/sbin/nginx - + expect fork respawn respawn limit 10 5 #oom never - + pre-start script $DAEMON -t if [ $? -ne 0 ] then exit $? fi end script - + exec $DAEMON diff --git a/nginx/users.sls b/nginx/users.sls index 34a6672..aadf49c 100644 --- a/nginx/users.sls +++ b/nginx/users.sls @@ -5,9 +5,14 @@ htpasswd: pkg.installed: - name: {{ nginx.apache_utils }} +touch {{ htauth }}: + cmd.run: + - creates: {{ htauth }} + make sure {{ htauth }} exists: - file.exists: + file.managed: - name: {{ htauth }} + - makedirs: True {% for name, user in pillar.get('users', {}).items() %} {% if user['webauth'] is defined -%} diff --git a/pillar.example b/pillar.example index 4399586..90b1797 100644 --- a/pillar.example +++ b/pillar.example @@ -1,19 +1,19 @@ -nginx: - install_from_source: True - use_upstart: True - use_sysvinit: False - user_auth_enabled: True - with_luajit: False - with_openresty: True - repo_version: development # Must be using ppa install by setting `repo_source = ppa` - set_real_ips: # NOTE: to use this, nginx must have http_realip module enabled - from_ips: - - 10.10.10.0/24 - real_ip_header: X-Forwarded-For - modules: - headers-more: - source: http://github.com/agentzh/headers-more-nginx-module/tarball/v0.21 - source_hash: sha1=dbf914cbf3f7b6cb7e033fa7b7c49e2f8879113b +# nginx: + install_from_source: True + use_upstart: True + use_sysvinit: False + user_auth_enabled: True + with_luajit: False + with_openresty: True + repo_version: development # Must be using ppa install by setting `repo_source = ppa` + set_real_ips: # NOTE: to use this, nginx must have http_realip module enabled + from_ips: + - 10.10.10.0/24 + real_ip_header: X-Forwarded-For + modules: + headers-more: + source: http://github.com/agentzh/headers-more-nginx-module/tarball/v0.21 + source_hash: sha1=dbf914cbf3f7b6cb7e033fa7b7c49e2f8879113b # ======== # nginx.ng @@ -21,28 +21,46 @@ nginx: nginx: ng: + # The following three `install_from_` options are mutually exclusive. If none is used, the distro's provided + # package will be installed. If one of the `install_from` option is set to `True`, the state will + # make sure the other two repos are removed. + + # Use the official's nginx repo binaries + install_from_repo: false + + # Use Phusionpassenger's repo to install nginx and passenger binaries + # Debian, Centos, Ubuntu and Redhat are currently available + install_from_phusionpassenger: false + # PPA install - install_from_ppa: True + install_from_ppa: false # Set to 'stable', 'development' (mainline), 'community', or 'nightly' for each build accordingly ( https://launchpad.net/~nginx ) ppa_version: 'stable' # Source install source_version: '1.10.0' source_hash: '' - + # These are usually set by grains in map.jinja + # Typically you can comment these out. lookup: - package: nginx-custom + package: nginx-custom (can be a list) service: nginx webuser: www-data conf_file: /etc/nginx/nginx.conf server_available: /etc/nginx/sites-available server_enabled: /etc/nginx/sites-enabled server_use_symlink: True + # If you install nginx+passenger from phusionpassenger in Debian, these values will probably be needed + passenger_package: libnginx-mod-http-passenger + passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf + # This is required for RedHat like distros (Amazon Linux) that don't follow semantic versioning for $releasever rh_os_releasever: '6' # Currently it can be used on rhel/centos/suse when installing from repo gpg_check: True + pid_file: /var/run/nginx.pid ### Prevent Rendering SLS error (map.jinja:149) if nginx.server.config.pid undefined (Ubuntu, etc) ### + # Source compilation is not currently a part of nginx.ng from_source: False @@ -57,19 +75,29 @@ nginx: enable: True # Whether or not the service will be enabled/running or dead opts: {} # this partially exposes parameters of service.running / service.dead + snippets: # You can use snippets to define often repeated configuration once and include it later + letsencrypt: # e.g. this can be included using "- include: 'snippets/letsencrypt.conf'" + - location ^~ /.well-known/acme-challenge/: + - proxy_pass: http://localhost:9999 + server: opts: {} # this partially exposes file.managed parameters as they relate to the main nginx.conf file # nginx.conf (main server) declarations # dictionaries map to blocks {} and lists cause the same declaration to repeat with different values - config: + config: + source_path: salt://path_to_nginx_conf_file/nginx.conf # IMPORTANT: This option is mutually exclusive with the rest of the + # options; if it is found other options (worker_processes: 4 and so + # on) are not processed and just upload the file from source worker_processes: 4 - pid: /run/nginx.pid + load_module: modules/ngx_http_lua_module.so # this will be passed very first in configuration; otherwise nginx will fail to start + pid: /var/run/nginx.pid # Directory location must exist events: worker_connections: 768 http: sendfile: 'on' include: + #### Note: Syntax issues in these files generate nginx [emerg] errors on startup. #### - /etc/nginx/mime.types - /etc/nginx/conf.d/*.conf - /etc/nginx/sites-enabled/* @@ -79,26 +107,33 @@ nginx: symlink_opts: {} # partially exposes file.symlink params when symlinking enabled sites rename_opts: {} # partially exposes file.rename params when not symlinking disabled/enabled sites managed_opts: {} # partially exposes file.managed params for managed server files - dir_opts: {} # partially exposes file.directory params for site available/enabled dirs + dir_opts: {} # partially exposes file.directory params for site available/enabled and snippets dirs # server declarations # servers will default to being placed in server_available managed: mysite: # relative pathname of the server file # may be True, False, or None where True is enabled, False, disabled, and None indicates no action - available_dir: /tmp/sites-available # an alternate directory (not sites-available) where this server may be found - enabled_dir: /tmp/sites-enabled # an alternate directory (not sites-enabled) where this server may be found - disabled_name: mysite.aint_on # an alternative disabled name to be use when not symlinking enabled: True + # Remove the site config file. Nice to clean up the conf.d (or sites-available). + # It also remove the symlink (if it is exists). + # The site MUST be disabled before delete it (if not the nginx is not reloaded). + deleted: True + ########### + ## Modify 'available_dir' AND 'enabled_dir' '/etc/nginx' location to alternative value. + ########### + available_dir: /etc/nginx/sites-available # an alternate directory (not sites-available) where this server may be found + enabled_dir: /etc/nginx/sites-enabled # an alternate directory (not sites-enabled) where this server may be found + disabled_name: mysite.aint_on # an alternative disabled name to be use when not symlinking overwrite: True # overwrite an existing server file or not - + # May be a list of config options or None, if None, no server file will be managed/templated # Take server directives as lists of dictionaries. If the dictionary value is another list of # dictionaries a block {} will be started with the dictionary key name config: - server: - server_name: localhost - - listen: + - listen: - 80 - default_server - index: @@ -109,7 +144,8 @@ nginx: - $uri - $uri/ =404 - test: something else - + - include 'snippets/letsencrypt.conf' + # The above outputs: # server { # server_name localhost; @@ -119,13 +155,44 @@ nginx: # try_files $uri $uri/ =404; # test something else; # } - # } + # } + mysite2: # Using source_path options to upload the file instead of templating all the file + enabled: True + available_dir: /etc/nginx/sites-available + enabled_dir: /etc/nginx/sites-enabled + config: + source_path: salt://path-to-site-file/mysite2 + # Below configuration becomes handy if you want to create custom configuration files + # for example if you want to create /usr/local/etc/nginx/http_options.conf with + # the following content: + + # sendfile on; + # tcp_nopush on; + # tcp_nodelay on; + # send_iowait 12000; + + http_options.conf: + enabled: True + available_dir: /usr/local/etc/nginx + enabled_dir: /usr/local/etc/nginx + config: + - sendfile: 'on' + - tcp_nopush: 'on' + - tcp_nodelay: 'on' + - send_iowait: 12000 + + certificates_path: '/etc/nginx/ssl' # Use this if you need to deploy below certificates in a custom path. # If you're doing SSL termination, you can deploy certificates this way. # The private one(s) should go in a separate pillar file not in version # control (or use encrypted pillar data). certificates: 'www.example.com': + + # choose one of: deploying this cert by pillar (e.g. in combination with ext_pillar and file_tree) + # public_cert_pillar: certs:example.com:fullchain.pem + # private_key_pillar: certs:example.com:privkey.pem + # or directly pasting the cert public_cert: | -----BEGIN CERTIFICATE----- (Your Primary SSL certificate: www.example.com.crt) @@ -140,3 +207,20 @@ nginx: -----BEGIN RSA PRIVATE KEY----- (Your Private Key: www.example.com.key) -----END RSA PRIVATE KEY----- + + dh_param: + 'mydhparam1.pem': | + -----BEGIN DH PARAMETERS----- + (Your custom DH prime) + -----END DH PARAMETERS----- + # or to generate one on-the-fly + 'mydhparam2.pem': + keysize: 2048 + + # Passenger configuration + # Default passenger configuration is provided, and will be deployed in + # /etc/nginx/conf.d/passenger.conf + passenger: + passenger_root: /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini + passenger_ruby: /usr/bin/ruby + passenger_instance_registry_dir: /var/run/passenger-instreg diff --git a/tests/pytests/apply-all-tests/__init__.py b/tests/pytests/apply-all-tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytests/apply-all-tests/test_000_apply_state.py b/tests/pytests/apply-all-tests/test_000_apply_state.py new file mode 100644 index 0000000..cf0b620 --- /dev/null +++ b/tests/pytests/apply-all-tests/test_000_apply_state.py @@ -0,0 +1,23 @@ +from subprocess import check_output +from unittest import TestCase + + +class ApplyStateTest(TestCase): + + def test_000_apply(self): + state_apply_response = check_output(["salt-call", "--local", "state.apply"]) + print('') + print('-' * 50) + print('state_apply_response:') + print(state_apply_response) + print('-' * 50) + print('') + + state_apply_response = state_apply_response.split('\n') + summary = state_apply_response[-8:] + failed = 0 + for line in summary: + if line.startswith('Failed:'): + failed = int(line.split(':').pop().strip()) + + self.assertEqual(failed, 0) diff --git a/tests/srv/salt/top.sls b/tests/srv/salt/top.sls new file mode 100644 index 0000000..9f754f2 --- /dev/null +++ b/tests/srv/salt/top.sls @@ -0,0 +1,3 @@ +base: + '*': + - nginx diff --git a/tools/filltmpl.py b/tools/filltmpl.py new file mode 100644 index 0000000..0bbeace --- /dev/null +++ b/tools/filltmpl.py @@ -0,0 +1,27 @@ +import os +import sys + +from jinja2 import Template + +# base/tests +dir_path = os.path.dirname(os.path.realpath(__file__)) + +# base +base_path = os.path.dirname(dir_path) + + +if __name__ == '__main__': + formula_name = sys.argv[1] + image_tag = sys.argv[2] + + template = Template( + open(os.path.join(dir_path, 'templates', 'Dockerfile.j2')).read() + ) + + dockerfile = template.render({ + 'formula_name': formula_name, + 'image_tag': image_tag + }) + + with open(os.path.join(base_path, 'Dockerfile.{}'.format(image_tag)), 'w') as fh: + fh.write(dockerfile) diff --git a/tools/run-tests.sh b/tools/run-tests.sh new file mode 100755 index 0000000..550fa58 --- /dev/null +++ b/tools/run-tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -ev + +test -z $2 && echo "Usage: ${0} FORMULA_NAME OS_ID" && exit 1 +export FORMULA_NAME=$1 +export OS_ID=$2 + + +function docker-run-pytest() { + docker run --rm \ + -v "$@":/opt/tests \ + --env=STAGE=TEST \ + -h "salt-testing-${OS_ID}" \ + --name "salt-testing-${OS_ID}" \ + -it ${FORMULA_NAME}:"salt-testing-${OS_ID}" \ + pytest -sv /opt/tests +} + +for i in $(find $PWD/tests/pytests/* -maxdepth 0 -type d); do + docker-run-pytest $i; +done diff --git a/tools/templates/Dockerfile.j2 b/tools/templates/Dockerfile.j2 new file mode 100644 index 0000000..5686069 --- /dev/null +++ b/tools/templates/Dockerfile.j2 @@ -0,0 +1,14 @@ +FROM simplyadrian/allsalt:{{ image_tag }} + +{% if 'debian' in image_tag or 'ubuntu' in image_tag -%} +RUN apt-get update && \ + apt-get install -y python-pip +{% endif %} + +RUN pip install pytest && \ + sed -i "s/#master: salt/master: localhost/g" /etc/salt/minion + +ADD tests/srv /srv +ADD {{ formula_name }} /srv/salt/{{ formula_name }} + +WORKDIR /srv/salt