Import moni Prometheus configuration #32
@ -3,15 +3,15 @@
 | 
				
			|||||||
skip_clone: true
 | 
					skip_clone: true
 | 
				
			||||||
 | 
					
 | 
				
			||||||
pipeline:
 | 
					pipeline:
 | 
				
			||||||
  commit_lint:
 | 
					  # commit_lint:
 | 
				
			||||||
    image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest
 | 
					  #   image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-gommit:latest
 | 
				
			||||||
    secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
 | 
					  #   secrets: [ci_netrc_username, ci_netrc_password, ci_netrc_machine]
 | 
				
			||||||
    when:
 | 
					  #   when:
 | 
				
			||||||
      event: [push]
 | 
					  #     event: [push]
 | 
				
			||||||
    commands:
 | 
					  #   commands:
 | 
				
			||||||
      - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting
 | 
					  #     - git clone --single-branch -b $CI_COMMIT_BRANCH $CI_REPO_LINK ../salt-libertacasa-commit-linting
 | 
				
			||||||
      - cd ../salt-libertacasa-commit-linting
 | 
					  #     - cd ../salt-libertacasa-commit-linting
 | 
				
			||||||
      - bin/lint-commits.pl production
 | 
					  #     - bin/lint-commits.pl production
 | 
				
			||||||
 | 
					
 | 
				
			||||||
  code_lint:
 | 
					  code_lint:
 | 
				
			||||||
    image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-lint:latest
 | 
					    image: registry.opensuse.org/home/crameleon/libertacasa/containers/containerfile/libertacasa/pipeline-lint:latest
 | 
				
			||||||
 | 
				
			|||||||
							
								
								
									
										110
									
								
								pillar/id/moni_lysergic_dev.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										110
									
								
								pillar/id/moni_lysergic_dev.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,110 @@
 | 
				
			|||||||
 | 
					prometheus:
 | 
				
			||||||
 | 
					  pkg:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      prometheus:
 | 
				
			||||||
 | 
					        config:
 | 
				
			||||||
 | 
					          alerting:
 | 
				
			||||||
 | 
					            alertmanagers:
 | 
				
			||||||
 | 
					              - static_configs:
 | 
				
			||||||
 | 
					                - targets:
 | 
				
			||||||
 | 
					                  - localhost:9093
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          rule_files:
 | 
				
			||||||
 | 
					            - /etc/prometheus/alerts/lysergic/*.yml
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          scrape_configs:
 | 
				
			||||||
 | 
					            - job_name: 'prometheus'
 | 
				
			||||||
 | 
					              static_configs:
 | 
				
			||||||
 | 
					              - targets: ['localhost:9090']
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            - job_name: 'node_exporters_lysergic'
 | 
				
			||||||
 | 
					              scrape_timeout: 1m
 | 
				
			||||||
 | 
					              scrape_interval: 5m
 | 
				
			||||||
 | 
					              file_sd_configs:
 | 
				
			||||||
 | 
					              - files:
 | 
				
			||||||
 | 
					                - '/etc/prometheus/targets/node-lysergic.json'
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            - job_name: 'blackbox-2xx'
 | 
				
			||||||
 | 
					              metrics_path: /probe
 | 
				
			||||||
 | 
					              params:
 | 
				
			||||||
 | 
					                module: [http_2xx]
 | 
				
			||||||
 | 
					              file_sd_configs:
 | 
				
			||||||
 | 
					              - files: ['/etc/prometheus/targets/blackbox-2xx*.json']
 | 
				
			||||||
 | 
					              relabel_configs:
 | 
				
			||||||
 | 
					              - source_labels: [__address__]
 | 
				
			||||||
 | 
					                target_label: __param_target
 | 
				
			||||||
 | 
					              - source_labels: [__param_target]
 | 
				
			||||||
 | 
					                target_label: instance
 | 
				
			||||||
 | 
					              - target_label: __address__
 | 
				
			||||||
 | 
					                replacement: 127.0.0.1:9115
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            - job_name: 'blackbox-3xx'
 | 
				
			||||||
 | 
					              metrics_path: /probe
 | 
				
			||||||
 | 
					              params:
 | 
				
			||||||
 | 
					                module: [http_3xx]
 | 
				
			||||||
 | 
					              file_sd_configs:
 | 
				
			||||||
 | 
					              - files: ['/etc/prometheus/targets/blackbox-3xx*.json']
 | 
				
			||||||
 | 
					              relabel_configs:
 | 
				
			||||||
 | 
					              - source_labels: [__address__]
 | 
				
			||||||
 | 
					                target_label: __param_target
 | 
				
			||||||
 | 
					              - source_labels: [__param_target]
 | 
				
			||||||
 | 
					                target_label: instance
 | 
				
			||||||
 | 
					              - target_label: __address__
 | 
				
			||||||
 | 
					                replacement: 127.0.0.1:9115
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            - job_name: 'certificate_exporter'
 | 
				
			||||||
 | 
					              static_configs:
 | 
				
			||||||
 | 
					              - targets: ['therapon.rigel.lysergic.dev:9793']
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					      alertmanager:
 | 
				
			||||||
 | 
					        config:
 | 
				
			||||||
 | 
					          route:
 | 
				
			||||||
 | 
					            group_by: ['alertname']
 | 
				
			||||||
 | 
					            group_wait: 10s
 | 
				
			||||||
 | 
					            group_interval: 10s
 | 
				
			||||||
 | 
					            repeat_interval: 1h
 | 
				
			||||||
 | 
					            receiver: 'smtp-local'
 | 
				
			||||||
 | 
					            routes:
 | 
				
			||||||
 | 
					            - receiver: 'lysergic'
 | 
				
			||||||
 | 
					          #    continue: false
 | 
				
			||||||
 | 
					              match:
 | 
				
			||||||
 | 
					               project: LYSERGIC
 | 
				
			||||||
 | 
					            - receiver: 'chillnet'
 | 
				
			||||||
 | 
					              match:
 | 
				
			||||||
 | 
					               project: CHILLNET
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          receivers:
 | 
				
			||||||
 | 
					          - name: 'smtp-local'
 | 
				
			||||||
 | 
					            email_configs:
 | 
				
			||||||
 | 
					            - to: 'system@lysergic.dev'
 | 
				
			||||||
 | 
					              from: 'alertmanager@moni.lysergic.dev'
 | 
				
			||||||
 | 
					              require_tls: false
 | 
				
			||||||
 | 
					          # !!! TO-DO
 | 
				
			||||||
 | 
					              smarthost: 'zz0.email:465'
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          - name: 'irc-libertacasa'
 | 
				
			||||||
 | 
					            webhook_configs:
 | 
				
			||||||
 | 
					            - url: 'http://127.0.0.1:2410/universe'
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          - name: 'lysergic'
 | 
				
			||||||
 | 
					            webhook_configs:
 | 
				
			||||||
 | 
					            - url: 'http://127.0.0.1:2410/universe'
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
 | 
					            - url: http://127.0.0.2:8081/prometheus/webhook
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
 | 
					            email_configs:
 | 
				
			||||||
 | 
					            - to: 'system@lysergic.dev'
 | 
				
			||||||
 | 
					              from: 'alertmanager@moni.lysergic.dev'
 | 
				
			||||||
 | 
					              require_tls: false
 | 
				
			||||||
 | 
					              smarthost: 'zz0.email:465'
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					          - name: 'chillnet'
 | 
				
			||||||
 | 
					            email_configs:
 | 
				
			||||||
 | 
					            - to: 'team@chillnet.org'
 | 
				
			||||||
 | 
					              from: 'alertmanager@moni.lysergic.dev'
 | 
				
			||||||
 | 
					              require_tls: false
 | 
				
			||||||
 | 
					              smarthost: 'zz0.email:465'
 | 
				
			||||||
 | 
					              send_resolved: yes
 | 
				
			||||||
							
								
								
									
										11
									
								
								pillar/role/monitoring/prometheus-alertmanager.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										11
									
								
								pillar/role/monitoring/prometheus-alertmanager.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,11 @@
 | 
				
			|||||||
 | 
					prometheus:
 | 
				
			||||||
 | 
					  wanted:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      - alertmanager
 | 
				
			||||||
 | 
					  pkg:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      alertmanager:
 | 
				
			||||||
 | 
					        config:
 | 
				
			||||||
 | 
					          global:
 | 
				
			||||||
 | 
					            resolve_timeout: 5m
 | 
				
			||||||
 | 
					
 | 
				
			||||||
							
								
								
									
										50
									
								
								pillar/role/monitoring/prometheus-exporter-blackbox.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										50
									
								
								pillar/role/monitoring/prometheus-exporter-blackbox.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,50 @@
 | 
				
			|||||||
 | 
					prometheus:
 | 
				
			||||||
 | 
					  wanted:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      - blackbox_exporter
 | 
				
			||||||
 | 
					  pkg:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      blackbox_exporter:
 | 
				
			||||||
 | 
					        config:
 | 
				
			||||||
 | 
					          modules:
 | 
				
			||||||
 | 
					            http_2xx:
 | 
				
			||||||
 | 
					              prober: http
 | 
				
			||||||
 | 
					              timeout: 15s
 | 
				
			||||||
 | 
					            http_post_2xx:
 | 
				
			||||||
 | 
					              prober: http
 | 
				
			||||||
 | 
					              http:
 | 
				
			||||||
 | 
					                method: POST
 | 
				
			||||||
 | 
					            http_3xx:
 | 
				
			||||||
 | 
					              prober: http
 | 
				
			||||||
 | 
					              timeout: 5s
 | 
				
			||||||
 | 
					              http:
 | 
				
			||||||
 | 
					                method: HEAD
 | 
				
			||||||
 | 
					                no_follow_redirects: true
 | 
				
			||||||
 | 
					                valid_status_codes: [301, 302]
 | 
				
			||||||
 | 
					            tcp_connect:
 | 
				
			||||||
 | 
					              prober: tcp
 | 
				
			||||||
 | 
					            ssh_banner:
 | 
				
			||||||
 | 
					              prober: tcp
 | 
				
			||||||
 | 
					              tcp:
 | 
				
			||||||
 | 
					                query_response:
 | 
				
			||||||
 | 
					                - expect: "^SSH-2.0-"
 | 
				
			||||||
 | 
					            irc_banner:
 | 
				
			||||||
 | 
					              prober: tcp
 | 
				
			||||||
 | 
					              tcp:
 | 
				
			||||||
 | 
					                query_response:
 | 
				
			||||||
 | 
					                - send: "NICK prober"
 | 
				
			||||||
 | 
					                - send: "USER prober prober prober :prober"
 | 
				
			||||||
 | 
					                - expect: "PING :([^ ]+)"
 | 
				
			||||||
 | 
					                  send: "PONG ${1}"
 | 
				
			||||||
 | 
					                - expect: "^:[^ ]+ 001"
 | 
				
			||||||
 | 
					            icmp:
 | 
				
			||||||
 | 
					              prober: icmp
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					firewalld:
 | 
				
			||||||
 | 
					  zones:
 | 
				
			||||||
 | 
					    internal:
 | 
				
			||||||
 | 
					      ports:
 | 
				
			||||||
 | 
					        - comment: 'Prometheus Blackbox Exporter'
 | 
				
			||||||
 | 
					          port: 9115
 | 
				
			||||||
 | 
					          protocol: tcp
 | 
				
			||||||
 | 
					
 | 
				
			||||||
							
								
								
									
										17
									
								
								pillar/role/monitoring/prometheus.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										17
									
								
								pillar/role/monitoring/prometheus.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,17 @@
 | 
				
			|||||||
 | 
					prometheus:
 | 
				
			||||||
 | 
					  wanted:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      - prometheus
 | 
				
			||||||
 | 
					  pkg:
 | 
				
			||||||
 | 
					    component:
 | 
				
			||||||
 | 
					      prometheus:
 | 
				
			||||||
 | 
					        config:
 | 
				
			||||||
 | 
					          global:
 | 
				
			||||||
 | 
					            scrape_interval: 15s
 | 
				
			||||||
 | 
					            evaluation_interval: 1m
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					firewalld:
 | 
				
			||||||
 | 
					  zones:
 | 
				
			||||||
 | 
					    internal:
 | 
				
			||||||
 | 
					      services:
 | 
				
			||||||
 | 
					        - prometheus
 | 
				
			||||||
@ -2,7 +2,7 @@ include:
 | 
				
			|||||||
  - firewalld
 | 
					  - firewalld
 | 
				
			||||||
  - profile.seccheck
 | 
					  - profile.seccheck
 | 
				
			||||||
  - profile.zypp
 | 
					  - profile.zypp
 | 
				
			||||||
  - profile.node_exporter
 | 
					  - profile.prometheus.node_exporter
 | 
				
			||||||
  - users
 | 
					  - users
 | 
				
			||||||
  - .ssh
 | 
					  - .ssh
 | 
				
			||||||
  - postfix.config
 | 
					  - postfix.config
 | 
				
			||||||
 | 
				
			|||||||
							
								
								
									
										18
									
								
								salt/profile/prometheus/targets.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								salt/profile/prometheus/targets.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,18 @@
 | 
				
			|||||||
 | 
					{%- set mypillar = salt['pillar.get']('profile:prometheus:targets') %}
 | 
				
			||||||
 | 
					{%- set targetsdir = '/etc/prometheus/targets' %}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					{%- if mypillar | length %}
 | 
				
			||||||
 | 
					{{ targetsdir }}:
 | 
				
			||||||
 | 
					  file.directory:
 | 
				
			||||||
 | 
					    - group: prometheus
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					{%- for group, nodes in mypillar.items() %}
 | 
				
			||||||
 | 
					{{ targetsdir }}/{{ group }}.json:
 | 
				
			||||||
 | 
					  file.serialize:
 | 
				
			||||||
 | 
					    - dataset: {{ nodes }}
 | 
				
			||||||
 | 
					    - serializer: json
 | 
				
			||||||
 | 
					{%- endfor %}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					{%- else %}
 | 
				
			||||||
 | 
					{%- do salt.log.debug('profile.prometheus: no targets defined') %}
 | 
				
			||||||
 | 
					{%- endif %}
 | 
				
			||||||
							
								
								
									
										2
									
								
								salt/role/monitoring/prometheus-alertmanager.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										2
									
								
								salt/role/monitoring/prometheus-alertmanager.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,2 @@
 | 
				
			|||||||
 | 
					include:
 | 
				
			||||||
 | 
					  - prometheus.config
 | 
				
			||||||
							
								
								
									
										2
									
								
								salt/role/monitoring/prometheus-exporter-blackbox.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										2
									
								
								salt/role/monitoring/prometheus-exporter-blackbox.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,2 @@
 | 
				
			|||||||
 | 
					include:
 | 
				
			||||||
 | 
					  - prometheus.config
 | 
				
			||||||
							
								
								
									
										3
									
								
								salt/role/monitoring/prometheus.sls
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										3
									
								
								salt/role/monitoring/prometheus.sls
									
									
									
									
									
										Normal file
									
								
							@ -0,0 +1,3 @@
 | 
				
			|||||||
 | 
					include:
 | 
				
			||||||
 | 
					  - prometheus.config
 | 
				
			||||||
 | 
					  - profile.prometheus.targets
 | 
				
			||||||
| 
						
							
	
	
	
	
	
	
	
	 
					
					Georg marked this conversation as resolved
					
				 
				 | 
				|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user
	
I’m not sure if this ordering is alright - in case Prometheus needs a restart upon changes in the targets we might want to either move the profile.prometheus.targets include up here or move the include to the profile itself and add a
watch_into the states there.