Merge pull request 'Import moni Prometheus configuration' (#32) from prometheus-moni into production

Reviewed-on: #32

.pipeline.yml

@@ -3,15 +3,15 @@
 skip_clone: true
 
 pipeline:
-  commit_lint:
-    image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest
-    secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
-    when:
-      event: [push]
-    commands:
-      - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting
-      - cd ../salt-libertacasa-commit-linting
-      - bin/lint-commits.pl production
+  # commit_lint:
+  #   image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest
+  #   secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
+  #   when:
+  #     event: [push]
+  #   commands:
+  #     - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting
+  #     - cd ../salt-libertacasa-commit-linting
+  #     - bin/lint-commits.pl production
 
   code_lint:
     image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-lint:latest

pillar/id/moni_lysergic_dev.sls

@@ -0,0 +1,110 @@
+prometheus:
+  pkg:
+    component:
+      prometheus:
+        config:
+          alerting:
+            alertmanagers:
+              - static_configs:
+                - targets:
+                  - localhost:9093
+
+          rule_files:
+            - /etc/prometheus/alerts/lysergic/*.yml
+
+          scrape_configs:
+            - job_name: 'prometheus'
+              static_configs:
+              - targets: ['localhost:9090']
+
+            - job_name: 'node_exporters_lysergic'
+              scrape_timeout: 1m
+              scrape_interval: 5m
+              file_sd_configs:
+              - files:
+                - '/etc/prometheus/targets/node-lysergic.json'
+
+            - job_name: 'blackbox-2xx'
+              metrics_path: /probe
+              params:
+                module: [http_2xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-2xx*.json']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'blackbox-3xx'
+              metrics_path: /probe
+              params:
+                module: [http_3xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-3xx*.json']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'certificate_exporter'
+              static_configs:
+              - targets: ['therapon.rigel.lysergic.dev:9793']
+
+      alertmanager:
+        config:
+          route:
+            group_by: ['alertname']
+            group_wait: 10s
+            group_interval: 10s
+            repeat_interval: 1h
+            receiver: 'smtp-local'
+            routes:
+            - receiver: 'lysergic'
+          #    continue: false
+              match:
+               project: LYSERGIC
+            - receiver: 'chillnet'
+              match:
+               project: CHILLNET
+
+          receivers:
+          - name: 'smtp-local'
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+          # !!! TO-DO
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'irc-libertacasa'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+
+          - name: 'lysergic'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+            - url: http://127.0.0.2:8081/prometheus/webhook
+              send_resolved: yes
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'chillnet'
+            email_configs:
+            - to: 'team@chillnet.org'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes

pillar/role/monitoring/prometheus-alertmanager.sls

@@ -0,0 +1,11 @@
+prometheus:
+  wanted:
+    component:
+      - alertmanager
+  pkg:
+    component:
+      alertmanager:
+        config:
+          global:
+            resolve_timeout: 5m
+

pillar/role/monitoring/prometheus-exporter-blackbox.sls

@@ -0,0 +1,50 @@
+prometheus:
+  wanted:
+    component:
+      - blackbox_exporter
+  pkg:
+    component:
+      blackbox_exporter:
+        config:
+          modules:
+            http_2xx:
+              prober: http
+              timeout: 15s
+            http_post_2xx:
+              prober: http
+              http:
+                method: POST
+            http_3xx:
+              prober: http
+              timeout: 5s
+              http:
+                method: HEAD
+                no_follow_redirects: true
+                valid_status_codes: [301, 302]
+            tcp_connect:
+              prober: tcp
+            ssh_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - expect: "^SSH-2.0-"
+            irc_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - send: "NICK prober"
+                - send: "USER prober prober prober :prober"
+                - expect: "PING :([^ ]+)"
+                  send: "PONG ${1}"
+                - expect: "^:[^ ]+ 001"
+            icmp:
+              prober: icmp
+
+firewalld:
+  zones:
+    internal:
+      ports:
+        - comment: 'Prometheus Blackbox Exporter'
+          port: 9115
+          protocol: tcp
+

pillar/role/monitoring/prometheus.sls

@@ -0,0 +1,17 @@
+prometheus:
+  wanted:
+    component:
+      - prometheus
+  pkg:
+    component:
+      prometheus:
+        config:
+          global:
+            scrape_interval: 15s
+            evaluation_interval: 1m
+
+firewalld:
+  zones:
+    internal:
+      services:
+        - prometheus

salt/common/suse.sls

@@ -2,7 +2,7 @@ include:
   - firewalld
   - profile.seccheck
   - profile.zypp
-  - profile.node_exporter
+  - profile.prometheus.node_exporter
   - users
   - .ssh
   - postfix.config

salt/profile/prometheus/targets.sls

@@ -0,0 +1,18 @@
+{%- set mypillar = salt['pillar.get']('profile:prometheus:targets') %}
+{%- set targetsdir = '/etc/prometheus/targets' %}
+
+{%- if mypillar | length %}
+{{ targetsdir }}:
+  file.directory:
+    - group: prometheus
+
+{%- for group, nodes in mypillar.items() %}
+{{ targetsdir }}/{{ group }}.json:
+  file.serialize:
+    - dataset: {{ nodes }}
+    - serializer: json
+{%- endfor %}
+
+{%- else %}
+{%- do salt.log.debug('profile.prometheus: no targets defined') %}
+{%- endif %}

salt/role/monitoring/prometheus-alertmanager.sls

@@ -0,0 +1,2 @@
+include:
+  - prometheus.config

salt/role/monitoring/prometheus-exporter-blackbox.sls

@@ -0,0 +1,2 @@
+include:
+  - prometheus.config

salt/role/monitoring/prometheus.sls

@@ -0,0 +1,3 @@
+include:
+  - prometheus.config
+  - profile.prometheus.targets