From 30144947402928c11affa1b70939dfc6c0ecbe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Fri, 21 Aug 2020 10:33:20 -0300 Subject: [PATCH 1/4] feat(debian): allow to install using OS packages --- prometheus/map.jinja | 2 +- prometheus/osfamilymap.yaml | 69 ++++++++++++++++++++++++++--- prometheus/package/repo/install.sls | 23 +++++++--- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/prometheus/map.jinja b/prometheus/map.jinja index 6212ada..f9d2478 100644 --- a/prometheus/map.jinja +++ b/prometheus/map.jinja @@ -30,9 +30,9 @@ {%- set uri = '%s/%s/releases/download/%s/%s'|format(p.pkg.uri, name, v.version, name) %} {%- set url = '%s-%s.%s-%s.tar.gz'|format(uri, v.version|replace('v',''), p.kernel, p.arch) %} {%- set dir = '%s-%s'|format(name, v.version) %} - {%- endif %} {%- do p.pkg.component[name].update({'path': p.dir.archive ~ '/' + dir }) %} {%- do p.pkg.component[name]['archive'].update({'name': p.dir.archive + '/' + dir, 'source': url}) %} + {%- endif %} {%- endfor %} {%- endif %} diff --git a/prometheus/osfamilymap.yaml b/prometheus/osfamilymap.yaml index 0630d04..b61465b 100644 --- a/prometheus/osfamilymap.yaml +++ b/prometheus/osfamilymap.yaml @@ -25,14 +25,73 @@ Debian: - cron use_upstream_repo: false use_upstream_package: false - use_upstream_archive: true - repo: - humanname: 'prometheus repository' - key_url: 'https://s3-eu-west-1.amazonaws.com/deb.robustperception.io/41EFC99D.gpg' - file: /etc/apt/sources.list.d/prometheus.list + use_upstream_archive: false component: + prometheus: + name: prometheus + alertmanager: + name: prometheus-alertmanager + apache_exporter: + name: prometheus-apache-exporter + bind_exporter: + name: prometheus-bind-exporter + bird_exporter: + name: prometheus-bird-exporter + blackbox_exporter: + name: prometheus-blackbox-exporter + hacluster_exporter: + name: prometheus-hacluster-exporter + haproxy_exporter: + name: prometheus-haproxy-exporter + homeplug_exporter: + name: prometheus-homeplug-exporter + ipmi_exporter: + name: prometheus-ipmi-exporter + libvirt_exporter: + name: prometheus-libvirt-exporter + mailexporter: + name: prometheus-mailexporter + mongodb_exporter: + name: prometheus-mongodb-exporter + mysqld_exporter: + name: prometheus-mysqld-exporter + nginx_exporter: + name: prometheus-nginx-exporter + nginx_vts_exporter: + name: prometheus-nginx-vts-exporter node_exporter: + name: prometheus-node-exporter args_file: /etc/default/prometheus-node-exporter + node_exporter_collectors: + name: prometheus-node-exporter-collectors + openstack_exporter: + name: prometheus-openstack-exporter + pgbouncer_exporter: + name: prometheus-pgbouncer-exporter + postfix_exporter: + name: prometheus-postfix-exporter + postgres_exporter: + name: prometheus-postgres-exporter + process_exporter: + name: prometheus-process-exporter + pushgateway: + name: prometheus-pushgateway + snmp_exporter: + name: prometheus-snmp-exporter + sql_exporter: + name: prometheus-sql-exporter + squid_exporter: + name: prometheus-squid-exporter + tplink_plug_exporter: + name: prometheus-tplink-plug-exporter + trafficserver_exporter: + name: prometheus-trafficserver-exporter + varnish_exporter: + name: prometheus-varnish-exporter + xmpp_alerts: + name: prometheus-xmpp-alerts + + exporters: node_exporter: textfile_collectors_dependencies: diff --git a/prometheus/package/repo/install.sls b/prometheus/package/repo/install.sls index 555d489..73ed856 100644 --- a/prometheus/package/repo/install.sls +++ b/prometheus/package/repo/install.sls @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- # vim: ft=sls -{%- set tplroot = tpldir.split('/')[0] %} -{%- from tplroot ~ "/map.jinja" import prometheus as p with context %} +{%- if grains.os_family == 'RedHat' %} - {%- if p.pkg.use_upstream_repo and 'repo' in p.pkg and p.pkg.repo %} - {%- from tplroot ~ "/files/macros.jinja" import format_kwargs with context %} + {%- set tplroot = tpldir.split('/')[0] %} + {%- from tplroot ~ "/map.jinja" import prometheus as p with context %} + + {%- if p.pkg.use_upstream_repo and 'repo' in p.pkg and p.pkg.repo %} + {%- from tplroot ~ "/files/macros.jinja" import format_kwargs with context %} prometheus-package-repo-install-pkgrepo-managed: pkgrepo.managed: @@ -16,6 +18,15 @@ prometheus-package-repo-install-pkgrepo-managed: - pattern: ' gpgkey2=' - repl: '\n ' - ignore_if_missing: True - - onlyif: {{ grains.os_family == 'RedHat' }} + {%- endif %} - {%- endif %} +{%- else %} + +prometheus-package-repo-install-pkgrepo-managed: + test.show_notification: + - name: Skipping repository configuration + - text: | + At the moment, there's no repo for {{ grains['os'] }} + See https://prometheus.io/download/ + +{%- endif %} From c5ad8575906dd180d354d996f9929d6a737d50c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Fri, 21 Aug 2020 10:34:29 -0300 Subject: [PATCH 2/4] test(packages): check when using repo or archives --- .../repo/controls/packages_spec.rb | 23 ++++++++++++++++--- .../repo/controls/repositories_spec.rb | 23 +++++++++++++++++++ test/salt/pillar/repo.sls | 5 ++-- 3 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 test/integration/repo/controls/repositories_spec.rb diff --git a/test/integration/repo/controls/packages_spec.rb b/test/integration/repo/controls/packages_spec.rb index fce7c13..8919c16 100644 --- a/test/integration/repo/controls/packages_spec.rb +++ b/test/integration/repo/controls/packages_spec.rb @@ -1,9 +1,26 @@ # frozen_string_literal: true -control 'prometheus package' do +case platform[:family] +when 'redhat' + packages = %w[ + prometheus2 + alertmanager + node_exporter + ] +when 'debian' + packages = %w[ + prometheus + prometheus-alertmanager + prometheus-node-exporter + ] +end + +control 'prometheus packages' do title 'should be installed' - describe package('prometheus2') do - it { should be_installed } + packages.each do |p| + describe package(p) do + it { should be_installed } + end end end diff --git a/test/integration/repo/controls/repositories_spec.rb b/test/integration/repo/controls/repositories_spec.rb new file mode 100644 index 0000000..c68b6b4 --- /dev/null +++ b/test/integration/repo/controls/repositories_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +control 'repositories' do + impact 0.6 + title 'Configure the repositories' + desc ' + Configure the Debian/RedHat repositories for the supported platforms. + ' + tag 'repositories', 'apt', 'yum' + ref 'Prometheus prerequisites - Section: Prometheus package repositories', url: 'https://prometheus.io/download' + + case os[:family] + when 'debian' + describe file('/etc/apt/sources.list.d/prometheus.list') do + it { should_not exist } + end + when 'redhat', 'centos' + describe yum.repo('prometheus') do + it { should exist } + it { should be_enabled } + end + end +end diff --git a/test/salt/pillar/repo.sls b/test/salt/pillar/repo.sls index 03473dd..46a3170 100644 --- a/test/salt/pillar/repo.sls +++ b/test/salt/pillar/repo.sls @@ -9,8 +9,9 @@ prometheus: - rust component: - prometheus - - alertmanager # not in debian repo, only archive + - alertmanager - node_exporter + - blackbox_exporter # - memcached_exporter # not in upstream repo, only archive exporters: @@ -29,7 +30,7 @@ prometheus: smartctl: /usr/sbin/smartctl pkg: - use_upstream_repo: true + use_upstream_repo: {{ false if grains.os_family|lower in ('debian',) else true }} use_upstream_archive: false clientlibs: From 68aaa3437894f0740b127cbb49508f5547c447a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Fri, 21 Aug 2020 10:46:48 -0300 Subject: [PATCH 3/4] docs(pillar.example): add some comments --- pillar.example | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pillar.example b/pillar.example index 1da1d0b..462dc82 100644 --- a/pillar.example +++ b/pillar.example @@ -8,8 +8,10 @@ prometheus: - haskell - rust component: + # List components (ie, exporters) using underscores and + # removing the 'prometheus' prefix - prometheus - - alertmanager # not in debian repo, only archive + - alertmanager - node_exporter # - memcached_exporter # not in upstream repo, only archive @@ -37,6 +39,14 @@ prometheus: golang: version: v1.6.0 component: + # If you use OS packages in Debian's family, components should have + # a 'name' variable stating the name of the package (it's generally + # something like `prometheus-component-with-dashes-replacing-underscores` + # ie, + # node_exporter: + # name: prometheus-node-exporter + # + # See prometheus/osfamilymap.yaml for more examples alertmanager: config: # yamllint disable-line rule:line-length From 1eeda220d1b311e2dc0dfdab4e82d7edb8712d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20B=C3=A9rtoli?= Date: Fri, 21 Aug 2020 10:47:35 -0300 Subject: [PATCH 4/4] ci(travis,kitchen): update matrix --- .pre-commit-config.yaml | 56 ++++++++++++++++ .rstcheck.cfg | 3 + .travis.yml | 31 +++++++-- .yamllint | 3 +- kitchen.yml | 141 ++++++++++++++++++++-------------------- 5 files changed, 157 insertions(+), 77 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .rstcheck.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f4893e6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +default_stages: [commit] +repos: + - repo: https://github.com/dafyddj/commitlint-pre-commit-hook + rev: v2.3.0 + hooks: + - id: commitlint + name: Check commit message using commitlint + description: Lint commit message against @commitlint/config-conventional rules + stages: [commit-msg] + additional_dependencies: ['@commitlint/config-conventional@8.3.4'] + - id: commitlint-travis + stages: [manual] + additional_dependencies: ['@commitlint/config-conventional@8.3.4'] + always_run: true + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 2.1.3 + hooks: + - id: rubocop + exclude: ^Gemfile\.lock$ + - id: shellcheck + name: Check shell scripts with shellcheck + exclude_types: [zsh] + args: [] + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.23.0 + hooks: + - id: yamllint + name: Check YAML syntax with yamllint + args: [--strict] + types: [file] + files: > + (?x)^( + \.salt-lint| + \.yamllint| + .*\.example| + test/.*\.sls| + .*\.(yaml|yml) + )$ + exclude: ^test/salt/.*/.*\.sls$ + - repo: https://github.com/warpnet/salt-lint + rev: v0.3.0 + hooks: + - id: salt-lint + name: Check Salt files using salt-lint + files: ^.*\.(sls|jinja|j2|tmpl|tst)$ + - repo: https://github.com/myint/rstcheck + rev: 3f929574 + hooks: + - id: rstcheck + name: Check reST files using rstcheck + args: [--report=warning] diff --git a/.rstcheck.cfg b/.rstcheck.cfg new file mode 100644 index 0000000..05856dc --- /dev/null +++ b/.rstcheck.cfg @@ -0,0 +1,3 @@ +[rstcheck] +report=error +ignore_language=rst diff --git a/.travis.yml b/.travis.yml index 496e2df..e08a807 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ jobs: # Run all of the linters in a single job - language: 'node_js' node_js: 'lts/*' + cache: + directories: + - $HOME/.cache/pre-commit env: 'Lint' name: 'Lint: salt-lint, yamllint, rubocop, shellcheck & commitlint' before_install: 'skip' @@ -54,6 +57,10 @@ jobs: - npm i -D @commitlint/config-conventional @commitlint/travis-cli - commitlint-travis + # Install and run `pre-commit` + - pip install pre-commit + - pre-commit run --all-files --verbose + - pre-commit run --hook-stage manual --verbose commitlint-travis ## Define the rest of the matrix based on Kitchen testing # Make sure the instances listed below match up with @@ -69,11 +76,10 @@ jobs: # to allow for comprehensive local testing # Ref: https://github.com/saltstack-formulas/template-formula/issues/118 # Ref: https://github.com/saltstack-formulas/template-formula/issues/121 - - env: INSTANCE=default-debian-10-master-py3 - # env: INSTANCE=default-ubuntu-2004-master-py3 + - env: INSTANCE=default-ubuntu-2004-master-py3 # - env: INSTANCE=default-ubuntu-1804-master-py3 - env: INSTANCE=default-centos-8-master-py3 - # env: INSTANCE=default-fedora-32-master-py3 + - env: INSTANCE=default-fedora-32-master-py3 # - env: INSTANCE=default-fedora-31-master-py3 # - env: INSTANCE=default-opensuse-leap-152-master-py3 # - env: INSTANCE=default-amazonlinux-2-master-py3 @@ -83,8 +89,8 @@ jobs: # - env: INSTANCE=default-centos-8-3000-3-py3 # - env: INSTANCE=default-centos-7-3000-3-py3 # - env: INSTANCE=default-fedora-31-3000-3-py3 - # env: INSTANCE=default-opensuse-leap-152-3000-3-py3 - # env: INSTANCE=default-amazonlinux-2-3000-3-py3 + - env: INSTANCE=default-opensuse-leap-152-3000-3-py3 + - env: INSTANCE=default-amazonlinux-2-3000-3-py3 # - env: INSTANCE=default-ubuntu-1804-3000-3-py2 # - env: INSTANCE=default-ubuntu-1604-3000-3-py2 # - env: INSTANCE=default-arch-base-latest-3000-3-py2 @@ -100,6 +106,9 @@ jobs: # - env: INSTANCE=default-centos-6-2019-2-py2 # - env: INSTANCE=default-amazonlinux-1-2019-2-py2 - env: INSTANCE=default-arch-base-latest-2019-2-py2 + # REPO + - env: INSTANCE=repo-debian-10-3000-3-py3 + - env: INSTANCE=repo-centos-7-3000-3-py3 ## Define the release stage that runs `semantic-release` - stage: 'release' @@ -126,3 +135,15 @@ jobs: edge: true # Run `semantic-release` script: 'npx semantic-release@15.14' + +# Notification options: `always`, `never` or `change` +notifications: + webhooks: + if: 'repo = saltstack-formulas/prometheus-formula' + urls: + - https://saltstack-formulas.zulipchat.com/api/v1/external/travis?api_key=HsIq3o5QmLxdnVCKF9is0FUIpkpAY79P&stream=CI&topic=saltstack-formulas%2Fprometheus-formula&ignore_pull_requests=true + on_success: always # default: always + on_failure: always # default: always + on_start: always # default: never + on_cancel: always # default: always + on_error: always # default: always diff --git a/.yamllint b/.yamllint index a1d0979..ad379ab 100644 --- a/.yamllint +++ b/.yamllint @@ -10,7 +10,7 @@ extends: default # 3. Any YAML files under directory `.kitchen/`, introduced during local testing ignore: | node_modules/ - test/**/states/**/*.sls + test/**/*.sls .kitchen/ bin/kitchen prometheus/osfamilymap.yaml @@ -24,7 +24,6 @@ yaml-files: - .yamllint # SaltStack Formulas additional settings - '*.example' - - test/**/*.sls rules: empty-values: diff --git a/kitchen.yml b/kitchen.yml index ff8442c..d47979f 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -15,20 +15,26 @@ platforms: - name: debian-10-master-py3 driver: image: saltimages/salt-master-py3:debian-10 + - name: ubuntu-2004-master-py3 + driver: + image: saltimages/salt-master-py3:ubuntu-20.04 - name: ubuntu-1804-master-py3 driver: image: saltimages/salt-master-py3:ubuntu-18.04 - name: centos-8-master-py3 driver: image: saltimages/salt-master-py3:centos-8 + - name: fedora-32-master-py3 + driver: + image: saltimages/salt-master-py3:fedora-32 - name: fedora-31-master-py3 driver: image: saltimages/salt-master-py3:fedora-31 - - name: opensuse-leap-151-master-py3 + - name: opensuse-leap-152-master-py3 driver: - image: netmanagers/salt-master-py3:opensuse-leap-15.1 + image: saltimages/salt-master-py3:opensuse-leap-15.2 run_command: /usr/lib/systemd/systemd - # Workaround to avoid intermittent failures on `opensuse-leap-15.1`: + # Workaround to avoid intermittent failures on `opensuse-leap-15.2`: # => SCP did not finish successfully (255): (Net::SCP::Error) transport: max_ssh_sessions: 1 @@ -36,6 +42,47 @@ platforms: driver: image: saltimages/salt-master-py3:amazonlinux-2 + ## SALT `3000.3` + - name: debian-10-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:debian-10 + - name: debian-9-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:debian-9 + - name: ubuntu-1804-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:ubuntu-18.04 + - name: centos-8-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:centos-8 + - name: centos-7-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:centos-7 + - name: fedora-31-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:fedora-31 + - name: opensuse-leap-152-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:opensuse-leap-15.2 + run_command: /usr/lib/systemd/systemd + # Workaround to avoid intermittent failures on `opensuse-leap-15.2`: + # => SCP did not finish successfully (255): (Net::SCP::Error) + transport: + max_ssh_sessions: 1 + - name: amazonlinux-2-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:amazonlinux-2 + - name: ubuntu-1804-3000-3-py2 + driver: + image: saltimages/salt-3000.3-py2:ubuntu-18.04 + - name: ubuntu-1604-3000-3-py2 + driver: + image: saltimages/salt-3000.3-py2:ubuntu-16.04 + - name: arch-base-latest-3000-3-py2 + driver: + image: saltimages/salt-3000.3-py2:arch-base-latest + run_command: /usr/lib/systemd/systemd + ## SALT `2019.2` - name: debian-10-2019-2-py3 driver: @@ -46,92 +93,42 @@ platforms: - name: ubuntu-1804-2019-2-py3 driver: image: saltimages/salt-2019.2-py3:ubuntu-18.04 + - name: ubuntu-1604-2019-2-py3 + driver: + image: saltimages/salt-2019.2-py3:ubuntu-16.04 - name: centos-8-2019-2-py3 driver: image: saltimages/salt-2019.2-py3:centos-8 + - name: centos-7-2019-2-py3 + driver: + image: saltimages/salt-2019.2-py3:centos-7 - name: fedora-31-2019-2-py3 driver: image: saltimages/salt-2019.2-py3:fedora-31 - - name: opensuse-leap-151-2019-2-py3 + - name: opensuse-leap-152-2019-2-py3 driver: - image: netmanagers/salt-2019.2-py3:opensuse-leap-15.1 + image: saltimages/salt-2019.2-py3:opensuse-leap-15.2 run_command: /usr/lib/systemd/systemd - # Workaround to avoid intermittent failures on `opensuse-leap-15.1`: + # Workaround to avoid intermittent failures on `opensuse-leap-15.2`: # => SCP did not finish successfully (255): (Net::SCP::Error) transport: max_ssh_sessions: 1 - - name: centos-7-2019-2-py2 - driver: - image: netmanagers/salt-2019.2-py2:centos-7 - name: amazonlinux-2-2019-2-py3 driver: image: saltimages/salt-2019.2-py3:amazonlinux-2 + - name: centos-6-2019-2-py2 + driver: + image: saltimages/salt-2019.2-py2:centos-6 + run_command: /sbin/init + - name: amazonlinux-1-2019-2-py2 + driver: + image: saltimages/salt-2019.2-py2:amazonlinux-1 + run_command: /sbin/init - name: arch-base-latest-2019-2-py2 driver: image: saltimages/salt-2019.2-py2:arch-base-latest run_command: /usr/lib/systemd/systemd - ## SALT `2018.3` - - name: fedora-30-2018-3-py3 - driver: - image: netmanagers/salt-2018.3-py3:fedora-30 - - name: debian-9-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:debian-9 - - name: ubuntu-1604-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:ubuntu-16.04 - - name: centos-7-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:centos-7 - - name: opensuse-leap-151-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:opensuse-leap-15.1 - run_command: /usr/lib/systemd/systemd - # Workaround to avoid intermittent failures on `opensuse-leap-15.1`: - # => SCP did not finish successfully (255): (Net::SCP::Error) - transport: - max_ssh_sessions: 1 - - name: amazonlinux-1-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:amazonlinux-1 - run_command: /sbin/init - - name: arch-base-latest-2018-3-py2 - driver: - image: netmanagers/salt-2018.3-py2:arch-base-latest - run_command: /usr/lib/systemd/systemd - - ## SALT `2017.7` - - name: debian-8-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:debian-8 - - name: ubuntu-1604-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:ubuntu-16.04 - - name: centos-6-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:centos-6 - run_command: /sbin/init - - name: fedora-30-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:fedora-30 - - name: opensuse-leap-151-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:opensuse-leap-15.1 - run_command: /usr/lib/systemd/systemd - # Workaround to avoid intermittent failures on `opensuse-leap-15.1`: - # => SCP did not finish successfully (255): (Net::SCP::Error) - transport: - max_ssh_sessions: 1 - - name: amazonlinux-1-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:amazonlinux-1 - run_command: /sbin/init - - name: arch-base-latest-2017-7-py2 - driver: - image: netmanagers/salt-2017.7-py2:arch-base-latest - run_command: /usr/lib/systemd/systemd - provisioner: name: salt_solo log_level: debug @@ -152,6 +149,8 @@ verifier: suites: - name: default + excludes: + - centos-6-2018-3-py2 provisioner: state_top: base: @@ -169,6 +168,8 @@ suites: inspec_tests: - path: test/integration/default - name: repo + excludes: + - centos-6-2018-3-py2 provisioner: state_top: base: