Merge pull request #39 from netmanagers/master

Allow to use OS packages in Debian family
2020-08-21 20:02:11 +02:00 · 2020-08-21 20:02:11 +02:00 · 3f15c88d32
commit 3f15c88d32
parent 0166fbdf6c 1eeda220d1
12 changed files with 296 additions and 95 deletions
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_stages: [commit]
+repos:
+  - repo: https://github.com/dafyddj/commitlint-pre-commit-hook
+    rev: v2.3.0
+    hooks:
+      - id: commitlint
+        name: Check commit message using commitlint
+        description: Lint commit message against @commitlint/config-conventional rules
+        stages: [commit-msg]
+        additional_dependencies: ['@commitlint/config-conventional@8.3.4']
+      - id: commitlint-travis
+        stages: [manual]
+        additional_dependencies: ['@commitlint/config-conventional@8.3.4']
+        always_run: true
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 2.1.3
+    hooks:
+      - id: rubocop
+        exclude: ^Gemfile\.lock$
+      - id: shellcheck
+        name: Check shell scripts with shellcheck
+        exclude_types: [zsh]
+        args: []
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.23.0
+    hooks:
+      - id: yamllint
+        name: Check YAML syntax with yamllint
+        args: [--strict]
+        types: [file]
+        files: >
+          (?x)^(
+              \.salt-lint|
+              \.yamllint|
+              .*\.example|
+              test/.*\.sls|
+              .*\.(yaml|yml)
+          )$
+        exclude: ^test/salt/.*/.*\.sls$
+  - repo: https://github.com/warpnet/salt-lint
+    rev: v0.3.0
+    hooks:
+      - id: salt-lint
+        name: Check Salt files using salt-lint
+        files: ^.*\.(sls|jinja|j2|tmpl|tst)$
+  - repo: https://github.com/myint/rstcheck
+    rev: 3f929574
+    hooks:
+      - id: rstcheck
+        name: Check reST files using rstcheck
+        args: [--report=warning]
--- a/.rstcheck.cfg
+++ b/.rstcheck.cfg
@ -0,0 +1,3 @@
+[rstcheck]
+report=error
+ignore_language=rst
--- a/.travis.yml
+++ b/.travis.yml
@ -31,6 +31,9 @@ jobs:
    # Run all of the linters in a single job
    - language: 'node_js'
      node_js: 'lts/*'
+      cache:
+        directories:
+          - $HOME/.cache/pre-commit
      env: 'Lint'
      name: 'Lint: salt-lint, yamllint, rubocop, shellcheck & commitlint'
      before_install: 'skip'
@ -54,6 +57,10 @@ jobs:
        - npm i -D @commitlint/config-conventional
                   @commitlint/travis-cli
        - commitlint-travis
+        # Install and run `pre-commit`
+        - pip install pre-commit
+        - pre-commit run --all-files --verbose
+        - pre-commit run --hook-stage manual --verbose commitlint-travis

    ## Define the rest of the matrix based on Kitchen testing
    # Make sure the instances listed below match up with
@ -69,11 +76,10 @@ jobs:
    #       to allow for comprehensive local testing
    #       Ref: https://github.com/saltstack-formulas/template-formula/issues/118
    #       Ref: https://github.com/saltstack-formulas/template-formula/issues/121
-    - env: INSTANCE=default-debian-10-master-py3
-    # env: INSTANCE=default-ubuntu-2004-master-py3
+    - env: INSTANCE=default-ubuntu-2004-master-py3
    # - env: INSTANCE=default-ubuntu-1804-master-py3
    - env: INSTANCE=default-centos-8-master-py3
-    # env: INSTANCE=default-fedora-32-master-py3
+    - env: INSTANCE=default-fedora-32-master-py3
    # - env: INSTANCE=default-fedora-31-master-py3
    # - env: INSTANCE=default-opensuse-leap-152-master-py3
    # - env: INSTANCE=default-amazonlinux-2-master-py3
@ -83,8 +89,8 @@ jobs:
    # - env: INSTANCE=default-centos-8-3000-3-py3
    # - env: INSTANCE=default-centos-7-3000-3-py3
    # - env: INSTANCE=default-fedora-31-3000-3-py3
-    # env: INSTANCE=default-opensuse-leap-152-3000-3-py3
-    # env: INSTANCE=default-amazonlinux-2-3000-3-py3
+    - env: INSTANCE=default-opensuse-leap-152-3000-3-py3
+    - env: INSTANCE=default-amazonlinux-2-3000-3-py3
    # - env: INSTANCE=default-ubuntu-1804-3000-3-py2
    # - env: INSTANCE=default-ubuntu-1604-3000-3-py2
    # - env: INSTANCE=default-arch-base-latest-3000-3-py2
@ -100,6 +106,9 @@ jobs:
    # - env: INSTANCE=default-centos-6-2019-2-py2
    # - env: INSTANCE=default-amazonlinux-1-2019-2-py2
    - env: INSTANCE=default-arch-base-latest-2019-2-py2
+    # REPO
+    - env: INSTANCE=repo-debian-10-3000-3-py3
+    - env: INSTANCE=repo-centos-7-3000-3-py3

    ## Define the release stage that runs `semantic-release`
    - stage: 'release'
@ -126,3 +135,15 @@ jobs:
        edge: true
        # Run `semantic-release`
        script: 'npx semantic-release@15.14'
+
+# Notification options: `always`, `never` or `change`
+notifications:
+  webhooks:
+    if: 'repo = saltstack-formulas/prometheus-formula'
+    urls:
+      - https://saltstack-formulas.zulipchat.com/api/v1/external/travis?api_key=HsIq3o5QmLxdnVCKF9is0FUIpkpAY79P&stream=CI&topic=saltstack-formulas%2Fprometheus-formula&ignore_pull_requests=true
+    on_success: always  # default: always
+    on_failure: always  # default: always
+    on_start: always    # default: never
+    on_cancel: always   # default: always
+    on_error: always    # default: always
--- a/.yamllint
+++ b/.yamllint
@ -10,7 +10,7 @@ extends: default
 # 3. Any YAML files under directory `.kitchen/`, introduced during local testing
 ignore: |
  node_modules/
-  test/**/states/**/*.sls
+  test/**/*.sls
  .kitchen/
  bin/kitchen
  prometheus/osfamilymap.yaml
@ -24,7 +24,6 @@ yaml-files:
  - .yamllint
  # SaltStack Formulas additional settings
  - '*.example'
-  - test/**/*.sls

 rules:
  empty-values:
--- a/kitchen.yml
+++ b/kitchen.yml
@ -15,20 +15,26 @@ platforms:
  - name: debian-10-master-py3
    driver:
      image: saltimages/salt-master-py3:debian-10
+  - name: ubuntu-2004-master-py3
+    driver:
+      image: saltimages/salt-master-py3:ubuntu-20.04
  - name: ubuntu-1804-master-py3
    driver:
      image: saltimages/salt-master-py3:ubuntu-18.04
  - name: centos-8-master-py3
    driver:
      image: saltimages/salt-master-py3:centos-8
+  - name: fedora-32-master-py3
+    driver:
+      image: saltimages/salt-master-py3:fedora-32
  - name: fedora-31-master-py3
    driver:
      image: saltimages/salt-master-py3:fedora-31
-  - name: opensuse-leap-151-master-py3
+  - name: opensuse-leap-152-master-py3
    driver:
-      image: netmanagers/salt-master-py3:opensuse-leap-15.1
+      image: saltimages/salt-master-py3:opensuse-leap-15.2
      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
    # => SCP did not finish successfully (255):  (Net::SCP::Error)
    transport:
      max_ssh_sessions: 1
@ -36,6 +42,47 @@ platforms:
    driver:
      image: saltimages/salt-master-py3:amazonlinux-2

+  ## SALT `3000.3`
+  - name: debian-10-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:debian-10
+  - name: debian-9-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:debian-9
+  - name: ubuntu-1804-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:ubuntu-18.04
+  - name: centos-8-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:centos-8
+  - name: centos-7-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:centos-7
+  - name: fedora-31-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:fedora-31
+  - name: opensuse-leap-152-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:opensuse-leap-15.2
+      run_command: /usr/lib/systemd/systemd
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
+    # => SCP did not finish successfully (255):  (Net::SCP::Error)
+    transport:
+      max_ssh_sessions: 1
+  - name: amazonlinux-2-3000-3-py3
+    driver:
+      image: saltimages/salt-3000.3-py3:amazonlinux-2
+  - name: ubuntu-1804-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:ubuntu-18.04
+  - name: ubuntu-1604-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:ubuntu-16.04
+  - name: arch-base-latest-3000-3-py2
+    driver:
+      image: saltimages/salt-3000.3-py2:arch-base-latest
+      run_command: /usr/lib/systemd/systemd
+
  ## SALT `2019.2`
  - name: debian-10-2019-2-py3
    driver:
@ -46,92 +93,42 @@ platforms:
  - name: ubuntu-1804-2019-2-py3
    driver:
      image: saltimages/salt-2019.2-py3:ubuntu-18.04
+  - name: ubuntu-1604-2019-2-py3
+    driver:
+      image: saltimages/salt-2019.2-py3:ubuntu-16.04
  - name: centos-8-2019-2-py3
    driver:
      image: saltimages/salt-2019.2-py3:centos-8
+  - name: centos-7-2019-2-py3
+    driver:
+      image: saltimages/salt-2019.2-py3:centos-7
  - name: fedora-31-2019-2-py3
    driver:
      image: saltimages/salt-2019.2-py3:fedora-31
-  - name: opensuse-leap-151-2019-2-py3
+  - name: opensuse-leap-152-2019-2-py3
    driver:
-      image: netmanagers/salt-2019.2-py3:opensuse-leap-15.1
+      image: saltimages/salt-2019.2-py3:opensuse-leap-15.2
      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
+    # Workaround to avoid intermittent failures on `opensuse-leap-15.2`:
    # => SCP did not finish successfully (255):  (Net::SCP::Error)
    transport:
      max_ssh_sessions: 1
-  - name: centos-7-2019-2-py2
-    driver:
-      image: netmanagers/salt-2019.2-py2:centos-7
  - name: amazonlinux-2-2019-2-py3
    driver:
      image: saltimages/salt-2019.2-py3:amazonlinux-2
+  - name: centos-6-2019-2-py2
+    driver:
+      image: saltimages/salt-2019.2-py2:centos-6
+      run_command: /sbin/init
+  - name: amazonlinux-1-2019-2-py2
+    driver:
+      image: saltimages/salt-2019.2-py2:amazonlinux-1
+      run_command: /sbin/init
  - name: arch-base-latest-2019-2-py2
    driver:
      image: saltimages/salt-2019.2-py2:arch-base-latest
      run_command: /usr/lib/systemd/systemd

-  ## SALT `2018.3`
-  - name: fedora-30-2018-3-py3
-    driver:
-      image: netmanagers/salt-2018.3-py3:fedora-30
-  - name: debian-9-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:debian-9
-  - name: ubuntu-1604-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:ubuntu-16.04
-  - name: centos-7-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:centos-7
-  - name: opensuse-leap-151-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:opensuse-leap-15.1
-      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
-    # => SCP did not finish successfully (255):  (Net::SCP::Error)
-    transport:
-      max_ssh_sessions: 1
-  - name: amazonlinux-1-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:amazonlinux-1
-      run_command: /sbin/init
-  - name: arch-base-latest-2018-3-py2
-    driver:
-      image: netmanagers/salt-2018.3-py2:arch-base-latest
-      run_command: /usr/lib/systemd/systemd
-
-  ## SALT `2017.7`
-  - name: debian-8-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:debian-8
-  - name: ubuntu-1604-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:ubuntu-16.04
-  - name: centos-6-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:centos-6
-      run_command: /sbin/init
-  - name: fedora-30-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:fedora-30
-  - name: opensuse-leap-151-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:opensuse-leap-15.1
-      run_command: /usr/lib/systemd/systemd
-    # Workaround to avoid intermittent failures on `opensuse-leap-15.1`:
-    # => SCP did not finish successfully (255):  (Net::SCP::Error)
-    transport:
-      max_ssh_sessions: 1
-  - name: amazonlinux-1-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:amazonlinux-1
-      run_command: /sbin/init
-  - name: arch-base-latest-2017-7-py2
-    driver:
-      image: netmanagers/salt-2017.7-py2:arch-base-latest
-      run_command: /usr/lib/systemd/systemd
-
 provisioner:
  name: salt_solo
  log_level: debug
@ -152,6 +149,8 @@ verifier:

 suites:
  - name: default
+    excludes:
+      - centos-6-2018-3-py2
    provisioner:
      state_top:
        base:
@ -169,6 +168,8 @@ suites:
      inspec_tests:
        - path: test/integration/default
  - name: repo
+    excludes:
+      - centos-6-2018-3-py2
    provisioner:
      state_top:
        base:
--- a/pillar.example
+++ b/pillar.example
@ -8,8 +8,10 @@ prometheus:
      - haskell
      - rust
    component:
+      # List components (ie, exporters) using underscores and
+      # removing the 'prometheus' prefix
      - prometheus
-      - alertmanager          # not in debian repo, only archive
+      - alertmanager
      - node_exporter
      # - memcached_exporter  # not in upstream repo, only archive

@ -37,6 +39,14 @@ prometheus:
      golang:
        version: v1.6.0
    component:
+      # If you use OS packages in Debian's family, components should have
+      # a 'name' variable stating the name of the package (it's generally
+      # something like `prometheus-component-with-dashes-replacing-underscores`
+      # ie,
+      # node_exporter:
+      #   name: prometheus-node-exporter
+      #
+      # See prometheus/osfamilymap.yaml for more examples
      alertmanager:
        config:
          # yamllint disable-line rule:line-length
--- a/prometheus/map.jinja
+++ b/prometheus/map.jinja
@ -30,9 +30,9 @@
            {%- set uri = '%s/%s/releases/download/%s/%s'|format(p.pkg.uri, name, v.version, name) %}
            {%- set url = '%s-%s.%s-%s.tar.gz'|format(uri, v.version|replace('v',''), p.kernel, p.arch) %}
            {%- set dir = '%s-%s'|format(name, v.version) %}
-        {%- endif %}
        {%- do p.pkg.component[name].update({'path': p.dir.archive ~ '/' + dir }) %}
        {%- do p.pkg.component[name]['archive'].update({'name': p.dir.archive + '/' + dir, 'source': url}) %}
+        {%- endif %}
    {%- endfor %}
 {%- endif %}

--- a/prometheus/osfamilymap.yaml
+++ b/prometheus/osfamilymap.yaml
@ -25,14 +25,73 @@ Debian:
      - cron
    use_upstream_repo: false
    use_upstream_package: false
-    use_upstream_archive: true
-    repo:
-      humanname: 'prometheus repository'
-      key_url: 'https://s3-eu-west-1.amazonaws.com/deb.robustperception.io/41EFC99D.gpg'
-      file: /etc/apt/sources.list.d/prometheus.list
+    use_upstream_archive: false
    component:
+      prometheus:
+        name: prometheus
+      alertmanager:
+        name: prometheus-alertmanager
+      apache_exporter:
+        name: prometheus-apache-exporter
+      bind_exporter:
+        name: prometheus-bind-exporter
+      bird_exporter:
+        name: prometheus-bird-exporter
+      blackbox_exporter:
+        name: prometheus-blackbox-exporter
+      hacluster_exporter:
+        name: prometheus-hacluster-exporter
+      haproxy_exporter:
+        name: prometheus-haproxy-exporter
+      homeplug_exporter:
+        name: prometheus-homeplug-exporter
+      ipmi_exporter:
+        name: prometheus-ipmi-exporter
+      libvirt_exporter:
+        name: prometheus-libvirt-exporter
+      mailexporter:
+        name: prometheus-mailexporter
+      mongodb_exporter:
+        name: prometheus-mongodb-exporter
+      mysqld_exporter:
+        name: prometheus-mysqld-exporter
+      nginx_exporter:
+        name: prometheus-nginx-exporter
+      nginx_vts_exporter:
+        name: prometheus-nginx-vts-exporter
      node_exporter:
+        name: prometheus-node-exporter
        args_file: /etc/default/prometheus-node-exporter
+      node_exporter_collectors:
+        name: prometheus-node-exporter-collectors
+      openstack_exporter:
+        name: prometheus-openstack-exporter
+      pgbouncer_exporter:
+        name: prometheus-pgbouncer-exporter
+      postfix_exporter:
+        name: prometheus-postfix-exporter
+      postgres_exporter:
+        name: prometheus-postgres-exporter
+      process_exporter:
+        name: prometheus-process-exporter
+      pushgateway:
+        name: prometheus-pushgateway
+      snmp_exporter:
+        name: prometheus-snmp-exporter
+      sql_exporter:
+        name: prometheus-sql-exporter
+      squid_exporter:
+        name: prometheus-squid-exporter
+      tplink_plug_exporter:
+        name: prometheus-tplink-plug-exporter
+      trafficserver_exporter:
+        name: prometheus-trafficserver-exporter
+      varnish_exporter:
+        name: prometheus-varnish-exporter
+      xmpp_alerts:
+        name: prometheus-xmpp-alerts
+
+
  exporters:
    node_exporter:
      textfile_collectors_dependencies:
--- a/prometheus/package/repo/install.sls
+++ b/prometheus/package/repo/install.sls
@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
 # vim: ft=sls

-{%- set tplroot = tpldir.split('/')[0] %}
-{%- from tplroot ~ "/map.jinja" import prometheus as p with context %}
+{%- if grains.os_family == 'RedHat' %}
+
+  {%- set tplroot = tpldir.split('/')[0] %}
+  {%- from tplroot ~ "/map.jinja" import prometheus as p with context %}

  {%- if p.pkg.use_upstream_repo and 'repo' in p.pkg and p.pkg.repo %}
    {%- from tplroot ~ "/files/macros.jinja" import format_kwargs with context %}
@ -16,6 +18,15 @@ prometheus-package-repo-install-pkgrepo-managed:
    - pattern: ' gpgkey2='
    - repl: '\n       '
    - ignore_if_missing: True
-    - onlyif: {{ grains.os_family == 'RedHat' }}
-
  {%- endif %}
+
+{%- else %}
+
+prometheus-package-repo-install-pkgrepo-managed:
+  test.show_notification:
+    - name: Skipping repository configuration
+    - text: |
+        At the moment, there's no repo for {{ grains['os'] }}
+        See https://prometheus.io/download/
+
+{%- endif %}
--- a/test/integration/repo/controls/packages_spec.rb
+++ b/test/integration/repo/controls/packages_spec.rb
@ -1,9 +1,26 @@
 # frozen_string_literal: true

-control 'prometheus package' do
+case platform[:family]
+when 'redhat'
+  packages = %w[
+    prometheus2
+    alertmanager
+    node_exporter
+  ]
+when 'debian'
+  packages = %w[
+    prometheus
+    prometheus-alertmanager
+    prometheus-node-exporter
+  ]
+end
+
+control 'prometheus packages' do
  title 'should be installed'

-  describe package('prometheus2') do
+  packages.each do |p|
+    describe package(p) do
      it { should be_installed }
    end
+  end
 end
--- a/test/integration/repo/controls/repositories_spec.rb
+++ b/test/integration/repo/controls/repositories_spec.rb
@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+control 'repositories' do
+  impact 0.6
+  title 'Configure the repositories'
+  desc '
+    Configure the Debian/RedHat repositories for the supported platforms.
+  '
+  tag 'repositories', 'apt', 'yum'
+  ref 'Prometheus prerequisites - Section: Prometheus package repositories', url: 'https://prometheus.io/download'
+
+  case os[:family]
+  when 'debian'
+    describe file('/etc/apt/sources.list.d/prometheus.list') do
+      it { should_not exist }
+    end
+  when 'redhat', 'centos'
+    describe yum.repo('prometheus') do
+      it { should exist }
+      it { should be_enabled }
+    end
+  end
+end
--- a/test/salt/pillar/repo.sls
+++ b/test/salt/pillar/repo.sls
@ -9,8 +9,9 @@ prometheus:
      - rust
    component:
      - prometheus
-      - alertmanager          # not in debian repo, only archive
+      - alertmanager
      - node_exporter
+      - blackbox_exporter
      # - memcached_exporter  # not in upstream repo, only archive

  exporters:
@ -29,7 +30,7 @@ prometheus:
          smartctl: /usr/sbin/smartctl

  pkg:
-    use_upstream_repo: true
+    use_upstream_repo: {{ false if grains.os_family|lower in ('debian',) else true }}
    use_upstream_archive: false

    clientlibs: