Moni: Read Blackbox targets as JSON

Use uniform JSON target files instead of a JSON/YAML mix.

Signed-off-by: Georg Pfuetzenreuter <mail@georg-pfuetzenreuter.net>

.pipeline.yml

@@ -1,9 +1,22 @@
+---
+# yamllint disable rule:line-length
 skip_clone: true
 
 pipeline:
+  lint:
+    image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-lint:latest
+    secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
+    when:
+      event: [push]
+    commands:
+      - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-linting
+      - cd ../salt-libertacasa-linting
+      - find . -type f \( -name '*.yaml' -o -name '*.yml' \) -exec yamllint -f colored -s {} +
+      - find . -name '*.sls' -exec salt-lint --severity -x 204 {} +
+
   check:
     image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline:latest
-    secrets: [ ci_netrc_username, ci_netrc_password, ci_netrc_machine ]
+    secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
     when:
       event: [push]
     commands:
@@ -29,5 +42,5 @@ pipeline:
       event: [push]
       instance: woodpecker-orpheus.intranet.squirrelcube.com
     commands:
-      #- rolesyncer
+      # - rolesyncer
       - bin/rolesyncer.py

pillar/formulas.yaml

@@ -1,3 +1,4 @@
+---
 - firewalld
 - keepalived
 - nginx

pillar/id/dericom02_rigel_lysergic_dev.sls

@@ -23,7 +23,7 @@ profile:
             NickServNick: viaduct
             NickServPassword: ${'secret_matterbridge:general:accounts:irc.libertacasa:NickServPassword'}
             ColorNicks: 'true'
-            Charset: utf8 
+            Charset: utf8
             MessageSplit: 'true'
             MessageQueue: 60
             UseRelayMsg: 'true'

pillar/id/deriweb01_rigel_lysergic_dev.sls

@@ -412,7 +412,7 @@ nginx:
             - location /:
                 - proxy_pass: http://media.takahe.rigel.lysergic.dev:8001
                 {{ takaheresolver }}
-          {#- despair.life is a second entry-point to social.liberta.casa instead of only a secondary domain in Takahe #} 
+          {#- despair.life is a second entry-point to social.liberta.casa instead of only a secondary domain in Takahe #}
           - server:
             {{ takahe_includes() }}
             - server_name: despair.life
@@ -436,7 +436,7 @@ nginx:
               - snippets/error
             - server_name: exhausted.life
             {{ takahe_gohome() }}
-            - location /.well-known/: 
+            - location /.well-known/:
                 - proxy_pass: {{ backend.takahe }}
                 - sub_filter_types: application/xml
                 - sub_filter: takahe.rigel.lysergic.dev:8000 exhausted.life

pillar/id/moni_lysergic_dev.sls

@@ -0,0 +1,110 @@
+prometheus:
+  pkg:
+    component:
+      prometheus:
+        config:
+          alerting:
+            alertmanagers:
+              - static_configs:
+                - targets:
+                  - localhost:9093
+
+          rule_files:
+            - /etc/prometheus/alerts/lysergic/*.yml
+
+          scrape_configs:
+            - job_name: 'prometheus'
+              static_configs:
+              - targets: ['localhost:9090']
+        
+            - job_name: 'node_exporters_lysergic'
+              scrape_timeout: 1m
+              scrape_interval: 5m
+              file_sd_configs:
+              - files:
+                - '/etc/prometheus/targets/node-lysergic.json'
+
+            - job_name: 'blackbox-2xx'
+              metrics_path: /probe
+              params:
+                module: [http_2xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-2xx*.json']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'blackbox-3xx'
+              metrics_path: /probe
+              params:
+                module: [http_3xx]
+              file_sd_configs:
+              - files: ['/etc/prometheus/targets/blackbox-3xx*.json']
+              relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 127.0.0.1:9115
+
+            - job_name: 'certificate_exporter'
+              static_configs:
+              - targets: ['therapon.rigel.lysergic.dev:9793']
+
+      alertmanager:
+        config:
+          route:
+            group_by: ['alertname']
+            group_wait: 10s
+            group_interval: 10s
+            repeat_interval: 1h
+            receiver: 'smtp-local'
+            routes:
+            - receiver: 'lysergic'
+          #    continue: false
+              match:
+               project: LYSERGIC
+            - receiver: 'chillnet'
+              match:
+               project: CHILLNET
+
+          receivers:
+          - name: 'smtp-local'
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+          # !!! TO-DO
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'irc-libertacasa'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+
+          - name: 'lysergic'
+            webhook_configs:
+            - url: 'http://127.0.0.1:2410/universe'
+              send_resolved: yes
+            - url: http://127.0.0.2:8081/prometheus/webhook
+              send_resolved: yes
+            email_configs:
+            - to: 'system@lysergic.dev'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes
+
+          - name: 'chillnet'
+            email_configs:
+            - to: 'team@chillnet.org'
+              from: 'alertmanager@moni.lysergic.dev'
+              require_tls: false
+              smarthost: 'zz0.email:465'
+              send_resolved: yes

pillar/role/monitoring/prometheus-alertmanager.sls

@@ -0,0 +1,11 @@
+prometheus:
+  wanted:
+    component:
+      - alertmanager
+  pkg:
+    component:
+      alertmanager:
+        config:
+          global:
+            resolve_timeout: 5m
+

pillar/role/monitoring/prometheus-exporter-blackbox.sls

@@ -0,0 +1,41 @@
+prometheus:
+  wanted:
+    component:
+      - blackbox_exporter
+  pkg:
+    component:
+      blackbox_exporter:
+        config:
+          modules:
+            http_2xx:
+              prober: http
+              timeout: 15s
+            http_post_2xx:
+              prober: http
+              http:
+                method: POST
+            http_3xx:
+              prober: http
+              timeout: 5s
+              http:
+                method: HEAD
+                no_follow_redirects: true
+                valid_status_codes: [301, 302]
+            tcp_connect:
+              prober: tcp
+            ssh_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - expect: "^SSH-2.0-"
+            irc_banner:
+              prober: tcp
+              tcp:
+                query_response:
+                - send: "NICK prober"
+                - send: "USER prober prober prober :prober"
+                - expect: "PING :([^ ]+)"
+                  send: "PONG ${1}"
+                - expect: "^:[^ ]+ 001"
+            icmp:
+              prober: icmp

pillar/role/monitoring/prometheus.sls

@@ -0,0 +1,11 @@
+prometheus:
+  wanted:
+    component:
+      - prometheus
+  pkg:
+    component:
+      prometheus:
+        config:
+          global:
+            scrape_interval: 15s
+            evaluation_interval: 1m

salt/common/suse.sls

@@ -2,7 +2,7 @@ include:
   - firewalld
   - profile.seccheck
   - profile.zypp
-  - profile.node_exporter
+  - profile.prometheus.node_exporter
   - users
   - .ssh
   - postfix.config

salt/profile/matterbridge/init.sls

@@ -46,7 +46,7 @@ matterbridge_{{ instance }}_mediadir:
     - user: matterbridge
     {#- to-do: implement some shared group #}
     - group: lighttpd
-    - mode: 750
+    - mode: '0750'
     - makedirs: True
 {%- endif %}
 

salt/profile/netcup_failover/init.sls

@@ -5,6 +5,6 @@ include:
   file.managed:
     - user: keepalived_script
     - group: wheel
-    - mode: 750
+    - mode: '0750'
     - template: jinja
     - source: salt://{{ slspath }}/files/failover.sh.j2

salt/profile/prometheus/targets.sls

@@ -0,0 +1,18 @@
+{%- set mypillar = salt['pillar.get']('profile:prometheus:targets') %}
+{%- set targetsdir = '/etc/prometheus/targets' %}
+
+{%- if mypillar | length %}
+{{ targetsdir }}:
+  file.directory:
+    - group: prometheus
+
+{%- for group, nodes in mypillar.items() %}
+{{ targetsdir }}/{{ group }}.json:
+  file.serialize:
+    - dataset: {{ nodes }}
+    - serializer: json
+{%- endfor %}
+
+{%- else %}
+{%- do salt.log.debug('profile.prometheus: no targets defined') %}
+{%- endif %}

salt/profile/salt/master.sls

@@ -17,7 +17,7 @@ salt_master_extension_modules_dirs:
       {%- endfor %}
     - user: root
     - group: salt
-    - mode: 0755
+    - mode: '0755'
 
 salt_master_extension_modules_bins:
   file.managed:
@@ -30,7 +30,7 @@ salt_master_extension_modules_bins:
       {%- endfor %}
     - user: root
     - group: salt
-    - mode: 0640
+    - mode: '0640'
     - require:
       - file: salt_master_extension_modules_dirs
 
@@ -72,7 +72,7 @@ salt_master_extra_packages:
       - requirepass {{ master_pillar['cache.redis.password'] }}
     - user: root
     - group: redis
-    - mode: 0640
+    - mode: '0640'
     - require:
       - pkg: redis
 
@@ -80,7 +80,7 @@ salt_master_extra_packages:
   file.directory:
     - user: redis
     - group: redis
-    - mode: 0750
+    - mode: '0750'
     - require:
       - pkg: redis
 

salt/profile/salt/minion.sls

@@ -1,5 +1,7 @@
 {%- set netbox_pillar = salt['pillar.get']('netbox') -%}
-{%- if 'custom_fields' in netbox_pillar and netbox_pillar['custom_fields']['salt_roles'] is not none and 'salt.syndic' in netbox_pillar['custom_fields']['salt_roles'] -%}
+{%- if 'custom_fields' in netbox_pillar
+    and netbox_pillar['custom_fields']['salt_roles'] is not none
+    and 'salt.syndic' in netbox_pillar['custom_fields']['salt_roles'] -%}
 {%- set master = salt['pillar.get']('salt:master:syndic_master') -%}
 {%- elif 'config_context' in netbox_pillar -%}
 {%- set master = netbox_pillar['config_context']['salt_master'] -%}

salt/role/monitoring/prometheus-alertmanager.sls

@@ -0,0 +1,2 @@
+include:
+  - prometheus.config

salt/role/monitoring/prometheus-exporter-blackbox.sls

@@ -0,0 +1,2 @@
+include:
+  - prometheus.config

salt/role/monitoring/prometheus.sls

@@ -0,0 +1,3 @@
+include:
+  - prometheus.config
+  - profile.prometheus.targets