Merge pull request 'cleanup interprotocol bridges' (#96) from cfg_matterbridge into production

Reviewed-on: #96
Reviewed-by: Georg Pfuetzenreuter <mail@georg-pfuetzenreuter.net>

bin/nbroles_to_grains.sh

@@ -1,4 +1,14 @@
 #!/usr/bin/env sh
 # This rewrites top-files to fetch roles from grains instead of our custom roles API. Useful for testing outside of the LibertaCasa infrastructure, but not recommended for production.
 
-sed -i "s/salt\['http.query'\].*/grains\['roles'\] -%}/" */top.sls
+potential_files=(*/top.sls salt/common/suse.sls)
+
+for file in ${potential_files[@]}
+do
+	if [ -f "$file" ]
+	then
+		files+="$file "
+	fi
+done
+
+sed -i "s/salt\['http.query'\].*/grains\['roles'\] -%}/" ${files[@]}

pillar/cluster/denc/web-proxy.sls

@@ -127,7 +127,7 @@ nginx:
             - client_max_body_size: 20M
             - modsecurity_rules: |-
                 '
-                SecRuleRemoveById 941160
+                SecRuleRemoveById 941160 949110
                 SecAction "id:900200, phase:1, nolog, pass, t:none, setvar:\'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH\'"
                 '
 
@@ -212,6 +212,19 @@ nginx:
             - error_log: /var/log/nginx/libsso_public.error.log
             - access_log: /var/log/nginx/libsso_public.access.log combined
 
+      agola.conf:
+        config:
+          - server:
+            - include:
+              - snippets/listen_ha
+              - snippets/tls_lysergic
+            - server_name: ci.lysergic.dev ci.git.com.de
+            - location /:
+              - proxy_pass: https://ci.lysergic.dev
+              - proxy_ssl_verify: 'on'
+              - include: snippets/proxy
+
+manage_firewall: True
 firewalld:
   zones:
     public:

pillar/global/init.sls

@@ -15,6 +15,7 @@ zypper:
   refreshdb_force: False
 
 firewalld:
+  FlushAllOnReload: 'yes'
   zones:
     internal:
       short: Internal

pillar/id/dencpod01_lysergic_dev.sls

@@ -0,0 +1 @@
+manage_firewall: True

pillar/id/dericom02_rigel_lysergic_dev.sls

@@ -1,4 +1,4 @@
-{%- set mediapath = '/srv/matterbridge/' -%}
+{%- set mediapath = '/var/lib/matterbridge/' -%}
 
 {%- macro discord_common() -%}
             AutoWebhooks: 'true'
@@ -39,7 +39,7 @@ profile:
             Debug: 'false'
           telegram.libertacasa:
             Token: ${'secret_matterbridge:general:accounts:telegram.libertacasa:Token'}
-            RemoteNickFormat: '<{NICK}> '
+            RemoteNickFormat: '[{PROTOCOL}] <{NICK}> '
             MessageFormat: HTMLNick
             Label: tg
             DisableWebPagePreview: 'true'
@@ -47,7 +47,7 @@ profile:
             Server: 192.168.0.110:2220
             Nick: LC
             RemoteNickFormat: '{PROTOCOL}:<{NICK}> '
-            Label: p
+            Label: ssh
           discord.23:
             Token: ${'secret_matterbridge:general:accounts:discord.23:Token'}
             Server: ${'secret_matterbridge:general:accounts:discord.23:Server'}
@@ -61,7 +61,6 @@ profile:
         gateways:
           libcasa:
             irc.libertacasa: '#libcasa'
-            sshchat.Psyched: sshchat
             xmpp.libertacasa: libcasa
           dev:
             irc.libertacasa: '#dev'
@@ -69,22 +68,19 @@ profile:
           lucy:
             irc.libertacasa: '#lucy'
             xmpp.libertacasa: lucy
+            telegram.libertacasa: '-1001795702961'
+            sshchat.Psyched: sshchat
           info:
             irc.libertacasa: '#libcasa.info'
             xmpp.libertacasa: libcasa.info
-            #telegram.libertacasa: '-1001518274267'
           chat:
-            irc.libertacasa: '#chai'
+            irc.libertacasa: '#chat'
             discord.23: chat
             xmpp.libertacasa: chat
-          dota:
+          petals:
-            irc.libertacasa: '#dotes'
+            irc.libertacasa: '#Petals'
-            discord.23: dotes
+            telegram.libertacasa: '-1001971550949'
-            xmpp.libertacasa: dota
+
-          aithunder:
-            irc.libertacasa: '#aithunder'
-          #  discord.aithunder: main-chat
-            xmpp.libertacasa: aithunder
 
       libertacasa-irc:
         general:
@@ -215,24 +211,61 @@ profile:
           nerds:
             irc.libertacasa: '#nerds'
             irc.nerds: '#nerds'
-          chillops:
-            irc.libertacasa: '#chillops'
-            irc.chillnet: '#chillops'
-            irc.stardust: '#chillnet-test'
           music:
             irc.libertacasa: '#music'
             irc.chillnet: '#music'
             irc.stardust: '#music'
+      chillnet:
+        general:
+          MediaDownloadSize: 1000000000
+          MediaDownloadPath: {{ mediapath }}chillnet
+          MediaServerDownload: https://up.chillnet.org
+        accounts:
+          irc.chillnet:
+            Server: irc.chillnet.org:6697
+            UseTLS: 'true'
+            UseSASL: 'true'
+            Nick: viaduct
+            NickServNick: viaduct
+            NickServPassword: ${'secret_matterbridge:chillnet:accounts:irc.chillnet:NickServPassword'}
+            ColorNicks: 'true'
+            Charset: utf8
+            MessageSplit: 'true'
+            MessageQueue: 60
+            UseRelayMsg: 'true'
+            RemoteNickFormat: '{NICK}/{LABEL}'
+          telegram.chillnet:
+            Token: ${'secret_matterbridge:chillnet:accounts:telegram.chillnet:Token'}
+            RemoteNickFormat: '&lt;{NICK}&gt; '
+            MessageFormat: HTMLNick
+            Label: tg
+            DisableWebPagePreview: 'true'
+          discord.23:
+            Token: ${'secret_matterbridge:general:accounts:discord.23:Token'}
+            Server: ${'secret_matterbridge:general:accounts:discord.23:Server'}
+            {{ discord_common() }}
+        gateways:
+          staff:
+            irc.chillnet: '#chillstaff'
+            telegram.chillnet: '-1001932699309'
+          devs:
+            irc.chillnet: '#chilldevs'
+            telegram.chillnet: '-1001778806358'
+            discord.23: chilldevs
 
   lighttpd:
     vhosts:
       matterbridge-general:
-        host: 'libertacasa-general\.matterbridge\.dericom02\.rigel\.lysergic\.dev'
+        host: 'libertacasa-general.matterbridge.dericom02.rigel.lysergic.dev'
         root: {{ mediapath }}libertacasa-general
       matterbridge-irc:
-        host: 'libertacasa-irc\.matterbridge\.dericom02\.rigel\.lysergic\.dev'
+        host: 'libertacasa-irc.matterbridge.dericom02.rigel.lysergic.dev'
         root: {{ mediapath }}libertacasa-irc
+      matterbridge-chillnet:
+        host: 'chillnet.matterbridge.dericom02.rigel.lysergic.dev'
+        root: {{ mediapath }}chillnet
 
+manage_firewall: True
 firewalld:
   zones:
     web:

pillar/id/derigsm01_rigel_lysergic_dev.sls

@@ -0,0 +1 @@
+manage_firewall: True

pillar/id/derimisc01_rigel_lysergic_dev.sls

@@ -12,3 +12,5 @@ tor:
       hostname: cr36xbvmgjwnfw4sly4kuc6c3ozhesjre3y5pggq5xdkkmbrq6dz4fad.onion
       hs_ed25519_public_key: PT0gZWQyNTUxOXYxLXB1YmxpYzogdHlwZTAgPT0AAAAUd+uGrDJs0tuSXjiqC8LbsnJJMSbx15jQ7calMDGHhw==
       hs_ed25519_secret_key: ${'secret_tor:hidden_services:irc:key'}
+
+manage_firewall: True

pillar/id/deriweb01_rigel_lysergic_dev.sls

@@ -44,15 +44,15 @@
                 - proxy_set_header: Host $http_host
                 - resolver: '{{ resolver }} ipv4=off valid=24h'
 {%- endmacro -%}
-{%- macro matterbridge_media(name) -%}
+{%- macro matterbridge_media(domain, name, tls='load') -%}
           - server:
             - include:
               - snippets/listen
-              - snippets/tls_load
+              - snippets/tls_{{ tls }}
               - snippets/tls
-            - server_name: {% if name == 'general' %}load.casa{%- else %}{{ name ~ '.load.casa' }}{%- endif %}
+            - server_name: {{ domain }}
             - location /:
-                - proxy_pass: http://libertacasa-{{ name }}.matterbridge.dericom02.rigel.lysergic.dev
+                - proxy_pass: http://{{ name }}.matterbridge.dericom02.rigel.lysergic.dev
 {%- endmacro -%}
 
 nginx:
@@ -71,6 +71,7 @@ nginx:
     {{ nginx_crtkeypair('meet', 'meet.com.de') | indent }}
     {{ nginx_crtkeypair('takahe', 'social.liberta.casa') | indent }}
     {{ nginx_crtkeypair('pub_sectigo', 'pub') | indent }}
+    {{ nginx_crtkeypair('up.chillnet.org', 'up.chillnet.org') | indent }}
 
     {#- locations shared between clearnet and Tor LibertaCasa servers #}
     libertacasa:
@@ -316,8 +317,9 @@ nginx:
 
       matterbridge.conf:
         config:
-          {{  matterbridge_media('general') }}
+          {{ matterbridge_media('load.casa', 'libertacasa-general') }}
-          {{  matterbridge_media('irc') }}
+          {{ matterbridge_media('irc.load.casa', 'libertacasa-irc') }}
+          {{ matterbridge_media('up.chillnet.org', 'chillnet', 'up.chillnet.org') }}
 
       meet.conf:
         config:
@@ -441,3 +443,4 @@ nginx:
                 - sub_filter_types: application/xml
                 - sub_filter: takahe.rigel.lysergic.dev:8000 exhausted.life
 
+manage_firewall: True

pillar/id/derutil01_rigel_lysergic_dev.sls

@@ -0,0 +1 @@
+manage_firewall: True

pillar/id/moni_lysergic_dev.sls

@@ -108,3 +108,16 @@ prometheus:
               require_tls: false
               smarthost: 'zz0.email:465'
               send_resolved: yes
+
+manage_firewall: True
+firewalld:
+  zones:
+    internal:
+      services:
+        - https
+      ports:
+        - comment: DNS Slave
+          port: 5353
+          protocol: tcp
+        - port: 5353
+          protocol: udp

pillar/id/orpheus_psyched_dev.sls

@@ -0,0 +1 @@
+manage_sshd: False

pillar/id/philia_rigel_lysergic_dev.sls

@@ -0,0 +1 @@
+manage_sshd: False

pillar/id/phoebe_lysergic_dev.sls

@@ -0,0 +1 @@
+manage_firewall: True

pillar/id/selene_psyched_dev.sls

@@ -0,0 +1 @@
+manage_sshd: False

pillar/id/theia_psyched_dev.sls

@@ -0,0 +1 @@
+manage_sshd: False

pillar/id/themis_lysergic_dev.sls

@@ -25,19 +25,19 @@ apache:
       DirectoryIndex: index.php
       Directory:
         /srv/www/BookStack/:
-          Options: 'Indexes FollowSymLinks -MultiViews'
+          Options: FollowSymLinks
           AllowOverride: None
           Require: all granted
           Formula_Append: |
             RewriteEngine On
-            RewriteCond '%{HTTP:Authorization} .'
+            RewriteCond %{HTTP:Authorization} .
-            RewriteCond '.* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]'
+            RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]
-            RewriteCond '%{REQUEST_FILENAME} !-d'
+            RewriteCond %{REQUEST_FILENAME} !-d
-            RewriteCond '%{REQUEST_URI} (.+)/$'
+            RewriteCond %{REQUEST_URI} (.+)/$
-            RewriteCond '^ %1 [L,R=301]'
+            RewriteRule ^ %1 [L,R=301]
-            RewriteCond '%{REQUEST_FILENAME} !-d'
+            RewriteCond %{REQUEST_FILENAME} !-d
-            RewriteCond '%{REQUEST_FILENAME} !-f'
+            RewriteCond %{REQUEST_FILENAME} !-f
-            RewriteCond '^ index.php [L]'
+            RewriteRule ^ index.php [L]
       {{ httpdformulaexcess() }}
       Formula_Append: |
         {{ httpdcommon('BookStack') }}
@@ -88,9 +88,9 @@ profile:
     saml2_email_attribute: email
     saml2_external_id_attribute: uid
     saml2_display_name_attributes: fullname
-    saml2_idp_entityid: https://libsso.net/realms/libertacasa
+    saml2_idp_entityid: https://libsso.net/realms/LibertaCasa
-    saml2_idp_sso: https://libsso.net/realms/libertacasa/protocol/saml
+    saml2_idp_sso: https://libsso.net/realms/LibertaCasa/protocol/saml
-    saml2_idp_slo: https://libsso.net/realms/libertacasa/protocol/saml
+    saml2_idp_slo: https://libsso.net/realms/LibertaCasa/protocol/saml
     saml2_idp_x509: ${'secret_bookstack:saml2_idp_x509'}
     saml2_autoload_metadata: false
     saml2_sp_x509: ${'secret_bookstack:saml2_sp_x509'}
@@ -142,6 +142,7 @@ profile:
       pwd: ${'secret_privatebin:model_options:pwd'}
       opt[12]: true
 
+manage_firewall: True
 firewalld:
   zones:
     backend:

pillar/id/thetrip_lysergic_dev.sls

@@ -0,0 +1,7 @@
+manage_firewall: True
+firewalld:
+  zones:
+    public:
+      services:
+        - http
+        - https

pillar/role/bookstack.sls

@@ -0,0 +1 @@
+# empty

pillar/role/privatebin.sls

@@ -0,0 +1 @@
+# empty

pillar/role/salt/master.sls

@@ -21,7 +21,7 @@ salt:
       - roots
       - git
     file_roots:
-      production:
+      __env__:
         {%- for formula in formulas %}
         - /srv/formulas/{{ formula }}-formula
         {%- endfor %}
@@ -30,6 +30,7 @@ salt:
       - https://git.com.de/LibertaCasa/salt.git:
         - user: ${'secret_salt:master:gitfs_remotes:LibertaCasa:user'}
         - password: ${'secret_salt:master:gitfs_remotes:LibertaCasa:password'}
+        - fallback: production
     ext_pillar:
       - netbox:
           api_url: ${'secret_salt:master:ext_pillar:netbox:api_url'}
@@ -59,6 +60,7 @@ salt:
     timeout: 20
     gather_job_timeout: 20
     keep_jobs: 30
+    ping_on_rotate: True
     user: ${'secret_salt:master:user'}
     syndic_user: ${'secret_salt:master:syndic_user'}
     cache.redis.unix_socket_path: ${'secret_salt:master:cache.redis.unix_socket_path'}

pillar/role/salt/minion.sls

@@ -3,4 +3,7 @@ salt:
   minion_remove_config: True
   minion:
     master_type: str
+    backup_mode: minion
+    cache_jobs: True
+    enable_gpu_grains: False
   saltenv: production

salt/common/openbsd.sls

@@ -0,0 +1 @@
+# Nothing yet

salt/common/ssh.sls

@@ -1,5 +1,6 @@
 include:
   - openssh.banner
+{%- if salt['pillar.get']('manage_sshd', True) %}
   - openssh.config
 
 /etc/ssh/user_ca:
@@ -10,3 +11,4 @@ include:
       {%- endfor -%}
     - require:
       - pkg: openssh
+{%- endif %}

salt/common/suse.sls

@@ -1,9 +1,16 @@
 include:
+  {#- drop pillar check after all firewall configurations have been imported #}
+  {%- if salt['pillar.get']('manage_firewall', False) %}
   - firewalld
+  {%- endif %}
   - profile.seccheck
   - profile.zypp
   - profile.prometheus.node_exporter
+  {%- if salt['cmd.run']("awk '/^passwd/{ print $2; exit }' /etc/nsswitch.conf") == 'sss' %}
+  {%- do salt.log.warning('Not configuring local users due to sss') %}
+  {%- else %}
   - users
+  {%- endif %}
   - .ssh
   - postfix.config
 
@@ -62,7 +69,6 @@ common_packages_remove:
       {#- we only use AutoYaST for the OS deployment #}
       - autoyast2
       - autoyast2-installation
-      - libX11-data
       - yast2-add-on
       - yast2-services-manager
       - yast2-slp

salt/profile/apache-httpd/init.sls

@@ -16,11 +16,8 @@
     - require:
       - file: {{ snippetsdir }}
     {#- formula dependencies #}
-    - require_in:
-      - module: apache-service-running-restart
-      - service: apache-service-running
     - watch_in:
-      - module: apache-service-running-reload
+      - service: apache-service-running
 {%- endfor %}
 {%- endif %}
 

salt/profile/bookstack/init.sls

@@ -10,7 +10,7 @@ bookstack_permissions:
   file.managed:
     - mode: '0640'
     - user: root
-    - group: wwwrun
+    - group: www
     - names:
       - {{ configfile }}
 
@@ -24,10 +24,10 @@ bookstack_permissions:
     - key_values:
         {%- macro condconf(option) %}
         {%- if option in mypillar -%}
-        {%- if mypillar[option] is string and mypillar[option].startswith('$') or mypillar[option] is number %}
+        {%- if ( mypillar[option] is string and mypillar[option].startswith('$') ) or mypillar[option] is number %}
         {%- set value = mypillar[option] %}
         {%- else %}
-        {%- set value = mypillar[option] | quote %}
+        {%- set value = "\"'" ~ mypillar[option] ~ "'\"" %}
         {%- endif %}
         {{ option | upper }}: {{ value }}
         {%- endif -%}

salt/profile/privatebin/init.sls

@@ -24,10 +24,10 @@ privatebin_clean:
     - sections:
         {%- macro conf(section, options) %}
         {%- for option in options.keys() -%}
-        {%- if mypillar[section][option] is string and mypillar[section][option].startswith('$') or mypillar[section][option] is number %}
+        {%- if ( mypillar[section][option] is string and mypillar[section][option].startswith('$') ) or mypillar[section][option] is number %}
         {%- set value = mypillar[section][option] -%}
         {%- else %}
-        {%- set value = mypillar[section][option] | quote -%}
+        {%- set value = "\"'" ~ mypillar[section][option] ~ "'\"" -%}
         {%- endif %}
           {{ option }}: {{ value }}
         {%- endfor -%}

salt/profile/salt/formulas.sls

@@ -0,0 +1,6 @@
+salt_master_formulas:
+  git.latest:
+    - name: https://git.com.de/LibertaCasa/salt-formulas.git
+    - target: /srv/formulas
+    - branch: production
+    - submodules: True

salt/profile/salt/master.sls

@@ -7,6 +7,7 @@
 
 include:
   - salt.master
+  - .formulas
 
 salt_master_extension_modules_dirs:
   file.directory:
@@ -34,20 +35,13 @@ salt_master_extension_modules_bins:
     - require:
       - file: salt_master_extension_modules_dirs
 
-salt_master_formulas:
-  git.latest:
-    - name: https://git.com.de/LibertaCasa/salt-formulas.git
-    - target: /srv/formulas
-    - branch: production
-    - submodules: True
-
 salt_master_extra_packages:
   pkg.installed:
     - names:
       - python3-ldap
       - python3-pynetbox
       - python3-redis
-      - redis
+      - redis7
       - salt-bash-completion
       - salt-fish-completion
       - salt-keydiff
@@ -74,7 +68,7 @@ salt_master_extra_packages:
     - group: redis
     - mode: '0640'
     - require:
-      - pkg: redis
+      - pkg: redis7
 
 /var/lib/redis/salt:
   file.directory:
@@ -82,19 +76,19 @@ salt_master_extra_packages:
     - group: redis
     - mode: '0750'
     - require:
-      - pkg: redis
+      - pkg: redis7
 
 salt_redis_service_enable:
   service.enabled:
     - name: {{ redis_service }}
     - require:
-      - pkg: redis
+      - pkg: redis7
 
 salt_redis_service_start:
   service.running:
     - name: {{ redis_service }}
     - require:
-      - pkg: redis
+      - pkg: redis7
     - watch:
       - file: {{ redis_config }}
 
@@ -102,7 +96,7 @@ salt_redis_membership:
   group.present:
     - name: redis
     - require:
-      - pkg: redis
+      - pkg: redis7
     - addusers:
       - {{ master_pillar['user'] }}
 {%- if pillar['secret_salt'] is defined %}