Merge pull request #39 from netmanagers/master

Allow to use OS packages in Debian family

.pre-commit-config.yaml

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_stages: [commit]
+repos:
+  - repo: https://github.com/dafyddj/commitlint-pre-commit-hook
+    rev: v2.3.0
+    hooks:
+      - id: commitlint
+        name: Check commit message using commitlint
+        description: Lint commit message against @commitlint/config-conventional rules
+        stages: [commit-msg]
+        additional_dependencies: ['@commitlint/config-conventional@8.3.4']
+      - id: commitlint-travis
+        stages: [manual]
+        additional_dependencies: ['@commitlint/config-conventional@8.3.4']
+        always_run: true
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 2.1.3
+    hooks:
+      - id: rubocop
+        exclude: ^Gemfile\.lock$
+      - id: shellcheck
+        name: Check shell scripts with shellcheck
+        exclude_types: [zsh]
+        args: []
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.23.0
+    hooks:
+      - id: yamllint
+        name: Check YAML syntax with yamllint
+        args: [--strict]
+        types: [file]
+        files: >
+          (?x)^(
+              \.salt-lint|
+              \.yamllint|
+              .*\.example|
+              test/.*\.sls|
+              .*\.(yaml|yml)
+          )$
+        exclude: ^test/salt/.*/.*\.sls$
+  - repo: https://github.com/warpnet/salt-lint
+    rev: v0.3.0
+    hooks:
+      - id: salt-lint
+        name: Check Salt files using salt-lint
+        files: ^.*\.(sls|jinja|j2|tmpl|tst)$
+  - repo: https://github.com/myint/rstcheck
+    rev: 3f929574
+    hooks:
+      - id: rstcheck
+        name: Check reST files using rstcheck
+        args: [--report=warning]

.rstcheck.cfg

@@ -0,0 +1,3 @@
+[rstcheck]
+report=error
+ignore_language=rst

.travis.yml

@@ -31,6 +31,9 @@ jobs:
     # Run all of the linters in a single job
     - language: 'node_js'
       node_js: 'lts/*'
+      cache:
+        directories:
+          - $HOME/.cache/pre-commit
       env: 'Lint'
       name: 'Lint: salt-lint, yamllint, rubocop, shellcheck & commitlint'
       before_install: 'skip'
@@ -54,6 +57,10 @@ jobs:
         - npm i -D @commitlint/config-conventional
                    @commitlint/travis-cli
         - commitlint-travis
+        # Install and run `pre-commit`
+        - pip install pre-commit
+        - pre-commit run --all-files --verbose
+        - pre-commit run --hook-stage manual --verbose commitlint-travis
 
     ## Define the rest of the matrix based on Kitchen testing
     # Make sure the instances listed below match up with
@@ -69,11 +76,10 @@ jobs:
     #       to allow for comprehensive local testing
     #       Ref: https://github.com/saltstack-formulas/template-formula/issues/118
     #       Ref: https://github.com/saltstack-formulas/template-formula/issues/121
-    - env: INSTANCE=default-debian-10-master-py3
+    - env: INSTANCE=default-ubuntu-2004-master-py3
-    # env: INSTANCE=default-ubuntu-2004-master-py3
     # - env: INSTANCE=default-ubuntu-1804-master-py3
     - env: INSTANCE=default-centos-8-master-py3
-    # env: INSTANCE=default-fedora-32-master-py3
+    - env: INSTANCE=default-fedora-32-master-py3
     # - env: INSTANCE=default-fedora-31-master-py3
     # - env: INSTANCE=default-opensuse-leap-152-master-py3
     # - env: INSTANCE=default-amazonlinux-2-master-py3
@@ -83,8 +89,8 @@ jobs:
     # - env: INSTANCE=default-centos-8-3000-3-py3
     # - env: INSTANCE=default-centos-7-3000-3-py3
     # - env: INSTANCE=default-fedora-31-3000-3-py3
-    # env: INSTANCE=default-opensuse-leap-152-3000-3-py3
+    - env: INSTANCE=default-opensuse-leap-152-3000-3-py3
-    # env: INSTANCE=default-amazonlinux-2-3000-3-py3
+    - env: INSTANCE=default-amazonlinux-2-3000-3-py3
     # - env: INSTANCE=default-ubuntu-1804-3000-3-py2
     # - env: INSTANCE=default-ubuntu-1604-3000-3-py2
     # - env: INSTANCE=default-arch-base-latest-3000-3-py2
@@ -100,6 +106,9 @@ jobs:
     # - env: INSTANCE=default-centos-6-2019-2-py2
     # - env: INSTANCE=default-amazonlinux-1-2019-2-py2
     - env: INSTANCE=default-arch-base-latest-2019-2-py2
+    # REPO
+    - env: INSTANCE=repo-debian-10-3000-3-py3
+    - env: INSTANCE=repo-centos-7-3000-3-py3
 
     ## Define the release stage that runs `semantic-release`
     - stage: 'release'
@@ -126,3 +135,15 @@ jobs:
         edge: true
         # Run `semantic-release`
         script: 'npx semantic-release@15.14'
+
+# Notification options: `always`, `never` or `change`
+notifications:
+  webhooks:
+    if: 'repo = saltstack-formulas/prometheus-formula'
+    urls:
+      - https://saltstack-formulas.zulipchat.com/api/v1/external/travis?api_key=HsIq3o5QmLxdnVCKF9is0FUIpkpAY79P&stream=CI&topic=saltstack-formulas%2Fprometheus-formula&ignore_pull_requests=true
+    on_success: always  # default: always
+    on_failure: always  # default: always
+    on_start: always    # default: never
+    on_cancel: always   # default: always
+    on_error: always    # default: always

.yamllint

@@ -10,7 +10,7 @@ extends: default
 # 3. Any YAML files under directory `.kitchen/`, introduced during local testing
 ignore: |
   node_modules/
-  test/**/states/**/*.sls
+  test/**/*.sls
   .kitchen/
   bin/kitchen
   prometheus/osfamilymap.yaml
@@ -24,7 +24,6 @@ yaml-files:
   - .yamllint
   # SaltStack Formulas additional settings
   - '*.example'
-  - test/**/*.sls
 
 rules:
   empty-values:

kitchen.yml

@@ -15,20 +15,26 @@ platforms:
   - name: debian-10-master-py3
     driver:
       image: saltimages/salt-master-py3:debian-10
+  - name: ubuntu-2004-master-py3
+    driver:
+      image: saltimages/salt-master-py3:ubuntu-20.04
   - name: ubuntu-1804-master-py3
     driver:
       image: saltimages/salt-master-py3:ubuntu-18.04
   - name: centos-8-master-py3
     driver:
       image: saltimages/salt-master-py3:centos-8
+  - name: fedora-32-master-py3
+    driver:
+      image: saltimages/salt-master-py3:fedora-32
   - name: fedora-31-master-py3
     driver:
       image: saltimages/salt-master-py3:fedora-31
-  - name: opensuse-leap-151-master-py3
+  - name: opensuse-leap-152-master-py3
     driver:
-      image: netmanagers/salt-master-py3:opensuse-leap-15.1
+      image: saltimages/salt-master-py3:opensuse-leap-15.2
       run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
     # => SCP did not finish successfully (255):  (Net::SCP::Error)
     transport:
       max_ssh_sessions: 1
@@ -36,6 +42,47 @@ platforms:
     driver:
       image: saltimages/salt-master-py3:amazonlinux-2
 
+  ## SALT `3000.3`
+  - name: debian-10-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:debian-10
+  - name: debian-9-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:debian-9
+  - name: ubuntu-1804-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:ubuntu-18.04
+  - name: centos-8-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:centos-8
+  - name: centos-7-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:centos-7
+  - name: fedora-31-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:fedora-31
+  - name: opensuse-leap-152-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:opensuse-leap-15.2
+      run_command: /usr/lib/systemd/systemd
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
+    # => SCP did not finish successfully (255):  (Net::SCP::Error)
+    transport:
+      max_ssh_sessions: 1
+  - name: amazonlinux-2-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:amazonlinux-2
+  - name: ubuntu-1804-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:ubuntu-18.04
+  - name: ubuntu-1604-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:ubuntu-16.04
+  - name: arch-base-latest-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:arch-base-latest
+      run_command: /usr/lib/systemd/systemd
+
   ## SALT `2019.2`
   - name: debian-10-2019-2-py3
     driver:
@@ -46,92 +93,42 @@ platforms:
   - name: ubuntu-1804-2019-2-py3
     driver:
       image: saltimages/salt-2019.2-py3:ubuntu-18.04
+  - name: ubuntu-1604-2019-2-py3
+    driver:
+      image: saltimages/salt-2019.2-py3:ubuntu-16.04
   - name: centos-8-2019-2-py3
     driver:
       image: saltimages/salt-2019.2-py3:centos-8
+  - name: centos-7-2019-2-py3
+    driver:
+      image: saltimages/salt-2019.2-py3:centos-7
   - name: fedora-31-2019-2-py3
     driver:
       image: saltimages/salt-2019.2-py3:fedora-31
-  - name: opensuse-leap-151-2019-2-py3
+  - name: opensuse-leap-152-2019-2-py3
     driver:
-      image: netmanagers/salt-2019.2-py3:opensuse-leap-15.1
+      image: saltimages/salt-2019.2-py3:opensuse-leap-15.2
       run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
     # => SCP did not finish successfully (255):  (Net::SCP::Error)
     transport:
       max_ssh_sessions: 1
-  - name: centos-7-2019-2-py2
-    driver:
-      image: netmanagers/salt-2019.2-py2:centos-7
   - name: amazonlinux-2-2019-2-py3
     driver:
       image: saltimages/salt-2019.2-py3:amazonlinux-2
+  - name: centos-6-2019-2-py2
+    driver:
+      image: saltimages/salt-2019.2-py2:centos-6
+      run_command: /sbin/init
+  - name: amazonlinux-1-2019-2-py2
+    driver:
+      image: saltimages/salt-2019.2-py2:amazonlinux-1
+      run_command: /sbin/init
   - name: arch-base-latest-2019-2-py2
     driver:
       image: saltimages/salt-2019.2-py2:arch-base-latest
       run_command: /usr/lib/systemd/systemd
 
-  ## SALT `2018.3`
-  - name: fedora-30-2018-3-py3
-    driver:
-      image: netmanagers/salt-2018.3-py3:fedora-30
-  - name: debian-9-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:debian-9
-  - name: ubuntu-1604-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:ubuntu-16.04
-  - name: centos-7-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:centos-7
-  - name: opensuse-leap-151-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:opensuse-leap-15.1
-      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
-    # => SCP did not finish successfully (255):  (Net::SCP::Error)
-    transport:
-      max_ssh_sessions: 1
-  - name: amazonlinux-1-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:amazonlinux-1
-      run_command: /sbin/init
-  - name: arch-base-latest-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:arch-base-latest
-      run_command: /usr/lib/systemd/systemd
-
-  ## SALT `2017.7`
-  - name: debian-8-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:debian-8
-  - name: ubuntu-1604-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:ubuntu-16.04
-  - name: centos-6-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:centos-6
-      run_command: /sbin/init
-  - name: fedora-30-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:fedora-30
-  - name: opensuse-leap-151-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:opensuse-leap-15.1
-      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
-    # => SCP did not finish successfully (255):  (Net::SCP::Error)
-    transport:
-      max_ssh_sessions: 1
-  - name: amazonlinux-1-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:amazonlinux-1
-      run_command: /sbin/init
-  - name: arch-base-latest-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:arch-base-latest
-      run_command: /usr/lib/systemd/systemd
-
 provisioner:
   name: salt_solo
   log_level: debug
@@ -152,6 +149,8 @@ verifier:
 
 suites:
   - name: default
+    excludes:
+      - centos-6-2018-3-py2
     provisioner:
       state_top:
         base:
@@ -169,6 +168,8 @@ suites:
       inspec_tests:
         - path: test/integration/default
   - name: repo
+    excludes:
+      - centos-6-2018-3-py2
     provisioner:
       state_top:
         base:

pillar.example

@@ -8,8 +8,10 @@ prometheus:
       - haskell
       - rust
     component:
+      # List components (ie, exporters) using underscores and
+      # removing the 'prometheus' prefix
       - prometheus
-      - alertmanager          # not in debian repo, only archive
+      - alertmanager
       - node_exporter
       # - memcached_exporter  # not in upstream repo, only archive
 
@@ -37,6 +39,14 @@ prometheus:
       golang:
         version: v1.6.0
     component:
+      # If you use OS packages in Debian's family, components should have
+      # a 'name' variable stating the name of the package (it's generally
+      # something like `prometheus-component-with-dashes-replacing-underscores`
+      # ie,
+      # node_exporter:
+      #   name: prometheus-node-exporter
+      #
+      # See prometheus/osfamilymap.yaml for more examples
       alertmanager:
         config:
           # yamllint disable-line rule:line-length

prometheus/map.jinja

@@ -30,9 +30,9 @@
             {%- set uri = '%s/%s/releases/download/%s/%s'|format(p.pkg.uri, name, v.version, name) %}
             {%- set url = '%s-%s.%s-%s.tar.gz'|format(uri, v.version|replace('v',''), p.kernel, p.arch) %}
             {%- set dir = '%s-%s'|format(name, v.version) %}
-        {%- endif %}
         {%- do p.pkg.component[name].update({'path': p.dir.archive ~ '/' + dir }) %}
         {%- do p.pkg.component[name]['archive'].update({'name': p.dir.archive + '/' + dir, 'source': url}) %}
+        {%- endif %}
     {%- endfor %}
 {%- endif %}
 

prometheus/osfamilymap.yaml

@@ -25,14 +25,73 @@ Debian:
       - cron
     use_upstream_repo: false
     use_upstream_package: false
-    use_upstream_archive: true
+    use_upstream_archive: false
-    repo:
-      humanname: 'prometheus repository'
-      key_url: 'https://s3-eu-west-1.amazonaws.com/deb.robustperception.io/41EFC99D.gpg'
-      file: /etc/apt/sources.list.d/prometheus.list
     component:
+      prometheus:
+        name: prometheus
+      alertmanager:
+        name: prometheus-alertmanager
+      apache_exporter:
+        name: prometheus-apache-exporter
+      bind_exporter:
+        name: prometheus-bind-exporter
+      bird_exporter:
+        name: prometheus-bird-exporter
+      blackbox_exporter:
+        name: prometheus-blackbox-exporter
+      hacluster_exporter:
+        name: prometheus-hacluster-exporter
+      haproxy_exporter:
+        name: prometheus-haproxy-exporter
+      homeplug_exporter:
+        name: prometheus-homeplug-exporter
+      ipmi_exporter:
+        name: prometheus-ipmi-exporter
+      libvirt_exporter:
+        name: prometheus-libvirt-exporter
+      mailexporter:
+        name: prometheus-mailexporter
+      mongodb_exporter:
+        name: prometheus-mongodb-exporter
+      mysqld_exporter:
+        name: prometheus-mysqld-exporter
+      nginx_exporter:
+        name: prometheus-nginx-exporter
+      nginx_vts_exporter:
+        name: prometheus-nginx-vts-exporter
       node_exporter:
+        name: prometheus-node-exporter
         args_file: /etc/default/prometheus-node-exporter
+      node_exporter_collectors:
+        name: prometheus-node-exporter-collectors
+      openstack_exporter:
+        name: prometheus-openstack-exporter
+      pgbouncer_exporter:
+        name: prometheus-pgbouncer-exporter
+      postfix_exporter:
+        name: prometheus-postfix-exporter
+      postgres_exporter:
+        name: prometheus-postgres-exporter
+      process_exporter:
+        name: prometheus-process-exporter
+      pushgateway:
+        name: prometheus-pushgateway
+      snmp_exporter:
+        name: prometheus-snmp-exporter
+      sql_exporter:
+        name: prometheus-sql-exporter
+      squid_exporter:
+        name: prometheus-squid-exporter
+      tplink_plug_exporter:
+        name: prometheus-tplink-plug-exporter
+      trafficserver_exporter:
+        name: prometheus-trafficserver-exporter
+      varnish_exporter:
+        name: prometheus-varnish-exporter
+      xmpp_alerts:
+        name: prometheus-xmpp-alerts
+
+
   exporters:
     node_exporter:
       textfile_collectors_dependencies:

prometheus/package/repo/install.sls

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 # vim: ft=sls
 
-{%- set tplroot = tpldir.split('/')[0] %}
+{%- if grains.os_family == 'RedHat' %}
-{%- from tplroot ~ "/map.jinja" import prometheus as p with context %}
 
-    {%- if p.pkg.use_upstream_repo and 'repo' in p.pkg and p.pkg.repo %}
+  {%- set tplroot = tpldir.split('/')[0] %}
-        {%- from tplroot ~ "/files/macros.jinja" import format_kwargs with context %}
+  {%- from tplroot ~ "/map.jinja" import prometheus as p with context %}
+
+  {%- if p.pkg.use_upstream_repo and 'repo' in p.pkg and p.pkg.repo %}
+    {%- from tplroot ~ "/files/macros.jinja" import format_kwargs with context %}
 
 prometheus-package-repo-install-pkgrepo-managed:
   pkgrepo.managed:
@@ -16,6 +18,15 @@ prometheus-package-repo-install-pkgrepo-managed:
     - pattern: ' gpgkey2='
     - repl: '\n       '
     - ignore_if_missing: True
-    - onlyif: {{ grains.os_family == 'RedHat' }}
+  {%- endif %}
 
-    {%- endif %}
+{%- else %}
+
+prometheus-package-repo-install-pkgrepo-managed:
+  test.show_notification:
+    - name: Skipping repository configuration
+    - text: |
+        At the moment, there's no repo for {{ grains['os'] }}
+        See https://prometheus.io/download/
+
+{%- endif %}

test/integration/repo/controls/packages_spec.rb

@@ -1,9 +1,26 @@
 # frozen_string_literal: true
 
-control 'prometheus package' do
+case platform[:family]
+when 'redhat'
+  packages = %w[
+    prometheus2
+    alertmanager
+    node_exporter
+  ]
+when 'debian'
+  packages = %w[
+    prometheus
+    prometheus-alertmanager
+    prometheus-node-exporter
+  ]
+end
+
+control 'prometheus packages' do
   title 'should be installed'
 
-  describe package('prometheus2') do
+  packages.each do |p|
-    it { should be_installed }
+    describe package(p) do
+      it { should be_installed }
+    end
   end
 end

test/integration/repo/controls/repositories_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+control 'repositories' do
+  impact 0.6
+  title 'Configure the repositories'
+  desc '
+    Configure the Debian/RedHat repositories for the supported platforms.
+  '
+  tag 'repositories', 'apt', 'yum'
+  ref 'Prometheus prerequisites - Section: Prometheus package repositories', url: 'https://prometheus.io/download'
+
+  case os[:family]
+  when 'debian'
+    describe file('/etc/apt/sources.list.d/prometheus.list') do
+      it { should_not exist }
+    end
+  when 'redhat', 'centos'
+    describe yum.repo('prometheus') do
+      it { should exist }
+      it { should be_enabled }
+    end
+  end
+end

test/salt/pillar/repo.sls

@@ -9,8 +9,9 @@ prometheus:
       - rust
     component:
       - prometheus
-      - alertmanager          # not in debian repo, only archive
+      - alertmanager
       - node_exporter
+      - blackbox_exporter
       # - memcached_exporter  # not in upstream repo, only archive
 
   exporters:
@@ -29,7 +30,7 @@ prometheus:
           smartctl: /usr/sbin/smartctl
 
   pkg:
-    use_upstream_repo: true
+    use_upstream_repo: {{ false if grains.os_family|lower in ('debian',) else true }}
     use_upstream_archive: false
 
     clientlibs: