common/borgbackup/templates/borgbackup-prometheus.sh

@@ -3,6 +3,9 @@
 export BORG_PASSPHRASE="{{repo_passphrase}}"
 export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
 
+# Force locale for correct formatting
+LANG=en_US.UTF-8
+
 # Metrics output file in the prometheus node-exporter directory 
 PROM_FILE="/var/lib/prometheus/node-exporter/borgbackup.prom"
 

common/docker_ldap/tasks/main.yml

@@ -22,8 +22,16 @@
     - docker-compose.yml
     - Dockerfile 
     - syncrepl_exporter.yml 
+  register: config
 
-- name: start openldap docker
+- name: "stop {{ servicename}} docker"
   docker_compose:
-    project_src: /srv/ldap/
+    project_src: "{{ basedir }}"
+    state: absent
+  when: config.changed
+
+- name: "start {{ servicename}} docker"
+  docker_compose:
+    project_src: "{{ basedir }}"
     state: present
+

common/docker_traefik/tasks/main.yml

@@ -52,4 +52,4 @@
   docker_compose:
     project_src: "{{ basedir }}"
     state: present
-    
\ No newline at end of file
+    

host_vars/verwaltung

@@ -86,35 +86,35 @@ alert:
 # Definition von Borgbackup Repositories 
 borgbackup_repos:
 
-  warpsrvint: 
+  # warpsrvint: 
 
-    # URL des Repos   
-    repo: "ssh://warpzone@192.168.0.201:22/data/warpzone/verwaltung"
+  #   # URL des Repos   
+  #   repo: "ssh://warpzone@192.168.0.201:22/data/warpzone/verwaltung"
     
-    # Repo-spezifische Optionen zum Aufruf von Borgbackup
-    # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
-    options: ""
+  #   # Repo-spezifische Optionen zum Aufruf von Borgbackup
+  #   # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
+  #   options: ""
 
-    # Compression Options, z,b. "zlib,5, "zstd,5"
-    compression: "zlib,5"
+  #   # Compression Options, z,b. "zlib,5, "zstd,5"
+  #   compression: "zlib,5"
 
-    # Prune Optionen 
-    prune: "--keep-within=2d --keep-daily=7 --keep-weekly=4 --keep-monthly=6"
+  #   # Prune Optionen 
+  #   prune: "--keep-within=2d --keep-daily=7 --keep-weekly=4 --keep-monthly=6"
     
-    # Backup Schedule 
-    weekday: "*"
-    hour: "10"
-    minute: "30"
-
-    #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
-    # directories:
-
-    # Monitoring
-    alert: true
-    warning_age: 26 
-    critical_age: 50
-    warning_count: 10
-    critical_count: 5
+  #   # Backup Schedule 
+  #   weekday: "*"
+  #   hour: "10"
+  #   minute: "30"
+
+  #   #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
+  #   # directories:
+
+  #   # Monitoring
+  #   alert: true
+  #   warning_age: 26 
+  #   critical_age: 50
+  #   warning_count: 10
+  #   critical_count: 5
 
   borgbase: 
 

host_vars/weatherwax

@@ -36,5 +36,6 @@ alert:
     crit: 8
   disks:
     - { mountpoint: "/",         warn: "10 GB", crit: "3 GB" }
-    - { mountpoint: "/mnt/data", warn: "10 GB", crit: "3 GB" }
+    # btrfs currently no data from node exporter 
+    # - { mountpoint: "/mnt/data", warn: "10 GB", crit: "3 GB" }
 

host_vars/webserver

@@ -71,9 +71,9 @@ alert:
     warn: 8
     crit: 16
   containers:
+    - { name: "coturn_coturn_1" }
     - { name: "dockerstats_app_1" }
     - { name: "dokuwiki_app_1" }
-    - { name: "coturn_coturn_1" }
     - { name: "gitlab_app_1" }
     - { name: "gitlab_dind_1" }
     - { name: "gitlab_runner_1" }
@@ -101,14 +101,15 @@ alert:
     - { name: "mail_mailman-core_1" }
     - { name: "mail_mailman-web_1" }
     - { name: "mail_mailman-nginx_1" }
+    - { name: "matrix_ma1sd_1" }
+    - { name: "matrix_db_1" }
+    - { name: "matrix_synapse_1" }
     - { name: "matterbridge_cw_1" }
     - { name: "matterbridge_wz_1" }
     - { name: "matterbridge_web_1" }
     - { name: "matterbridge_restarter_1" }
-    - { name: "matrix_ma1sd_1" }
-    - { name: "matrix_db_1" }
-    - { name: "matrix_synapse_1" }
     - { name: "traefik_app_1" }
+    - { name: "vpnserver_app_1" }
     - { name: "warpapi_app_1" }
     - { name: "wordpress_app_1" }
     - { name: "wordpress_db_1" }
@@ -125,35 +126,35 @@ alert:
 # Definition von Borgbackup Repositories 
 borgbackup_repos:
 
-  warpsrvint: 
+  # warpsrvint: 
 
-    # URL des Repos   
-    repo: "ssh://warpzone@192.168.0.201:22/data/warpzone/webserver"
+  #   # URL des Repos   
+  #   repo: "ssh://warpzone@192.168.0.201:22/data/warpzone/webserver"
     
-    # Repo-spezifische Optionen zum Aufruf von Borgbackup
-    # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
-    options: ""
+  #   # Repo-spezifische Optionen zum Aufruf von Borgbackup
+  #   # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
+  #   options: ""
 
-    # Compression Options, z,b. "zlib,5, "zstd,5"
-    compression: "zlib,5"
+  #   # Compression Options, z,b. "zlib,5, "zstd,5"
+  #   compression: "zlib,5"
 
-    # Prune Optionen 
-    prune: "--keep-within=2d --keep-daily=7 --keep-weekly=4 --keep-monthly=6"
+  #   # Prune Optionen 
+  #   prune: "--keep-within=2d --keep-daily=7 --keep-weekly=4 --keep-monthly=6"
     
-    # Backup Schedule 
-    weekday: "*"
-    hour: "6"
-    minute: "0"
-
-    #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
-    # directories:
-
-    # Monitoring
-    alert: true
-    warning_age: 26 
-    critical_age: 50
-    warning_count: 10
-    critical_count: 5
+  #   # Backup Schedule 
+  #   weekday: "*"
+  #   hour: "6"
+  #   minute: "0"
+
+  #   #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
+  #   # directories:
+
+  #   # Monitoring
+  #   alert: true
+  #   warning_age: 26 
+  #   critical_age: 50
+  #   warning_count: 10
+  #   critical_count: 5
 
   borgbase: 
 

hosts.yml

@@ -20,7 +20,7 @@ prod:
 
     vms:
       children:
-        weatherwax:
+        weatherwax-vms:
           hosts:
             # Server für interne Dienste
             # Container auf dem internen Proxmox Server
@@ -34,7 +34,7 @@ prod:
             carrot:
               ansible_ssh_host: 192.168.0.202
 
-        tiffany:
+        tiffany-vms:
           hosts:
             # Öffentlicher Webserver Warpzone
             # VM auf Tiffany

site.yml

@@ -153,7 +153,8 @@
       }
     - {
         role: common/docker_ldap, tags: ldap,
-        servicename: "ldap",
+        servicename: ldap,
+        basedir: /srv/ldap,
         domain: "ldap.warpzone.ms"
       }
     - { 
@@ -172,7 +173,9 @@
     - { 
         role: webserver/docker_dokuwiki, tags: dokuwiki,
         servicename: "dokuwiki",
-        domain: "wiki.warpzone.ms"
+        domain: "wiki.warpzone.ms",
+        basedir: /srv/dokuwiki,
+        healthchecks_url: "https://hc-ping.com/038adcfe-05bf-45b4-919b-88b69aab8844"
       }
     - { 
         role: webserver/docker_gitlab, tags: gitlab,
@@ -186,7 +189,7 @@
         basedir: /srv/icinga, 
         domain: icinga.warpzone.ms,
         api_port: 5665,
-        mysql_port: 33306, 
+        mysql_port: 33306
       }
     - { 
         role: webserver/docker_hackmd, tags: hackmd,

webserver/docker_dokuwiki/tasks/main.yml

@@ -9,23 +9,19 @@
     path: "{{item}}"
     state: "directory"
   with_items:
-    - /srv/dokuwiki/
-    - /srv/dokuwiki/data
-    - /srv/dokuwiki/pdftemplate
+    - "{{ basedir }}"
+    - "{{ basedir }}/data"
+    - "{{ basedir }}/pdftemplate"
 
 - name: Docker Compose Konfig-Datei erstellen
   template:
     src: "{{item}}"
-    dest: "/srv/dokuwiki/{{item}}"
+    dest: "{{ basedir }}/{{item}}"
   with_items:
     - docker-compose.yml
     - Dockerfile
     - sendmail_plenum.py
-
-- name: start dokuwiki docker
-  docker_compose:
-    project_src: /srv/dokuwiki/
-    state: present
+  register: config
 
 - name: Cronjob für Mailversand Plenumsmail
   cron: 
@@ -33,5 +29,16 @@
     weekday: "0" 
     hour: "20" 
     minute: "0" 
-    job: "/usr/bin/python3 /srv/dokuwiki/sendmail_plenum.py"
+    job: "/usr/bin/python3 {{ basedir }}/sendmail_plenum.py"
     disabled: false
+
+- name: "stop {{ servicename}} docker"
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: absent
+  when: config.changed
+
+- name: "start {{ servicename}} docker"
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: present

webserver/docker_dokuwiki/templates/sendmail_plenum.py

@@ -2,73 +2,76 @@ import smtplib
 import datetime
 from email.message import EmailMessage
 from email.utils import formatdate
-
-with open("/srv/dokuwiki/data/data/pages/intern/diskussionsthemen.txt") as fp:
-    raw = fp.read().split('\n')
-
-    # Extract Themen 
-    firstline = 0
-    lastline = 0
-    for index, line in enumerate(raw):
-        # Start marker
-        if "====== 1." in line:
-            firstline = index + 1
-            continue
+import requests
+
+def do_work():
+    with open("/srv/dokuwiki/data/data/pages/intern/diskussionsthemen.txt") as fp:
+        raw = fp.read().split('\n')
+
+        # Extract Themen 
+        firstline = 0
+        lastline = 0
+        for index, line in enumerate(raw):
+            # Start marker
+            if "====== 1." in line:
+                firstline = index + 1
+                continue
+            
+            # End marker
+            if "====== 2." in line:
+                lastline = index
+                break
         
-        # End marker
-        if "====== 2." in line:
-            lastline = index
-            break
-    
-    # Remove empty lines from topics
-    topics = []
-    for extracted in raw[firstline:lastline-1]:
-        if extracted != '':
-            topics.append(extracted)
-
-    # There are no topics - just exit
-    if (len(topics) == 0):
-        exit()
-
-    # calculate date of next tuesday
-    d = datetime.date.today()
-    while d.strftime('%a') != 'Tue':
-        d += datetime.timedelta(1)
-
-    # mail template 
-    mail = f"""
-Liebe Zonies,
-    
-es gibt wieder Themen die im Plenum besprochen werden sollen. Der nächste
-Reguläre Termin ist am nächsten Dienstag ({str(d)}) um 20:00.
-Im Wiki (https://wiki.warpzone.ms/intern:diskussionsthemen#diskussionsthemen_naechstes_plenum_automatische_einladung)
-stehen folgende Themen:
-
-{chr(10).join(topics)}
-
-Die Teilnahme ist natürlich auch remote möglich unter
-https://jitsi.dorf-post.de/warpzone-plenum möglich.
-
-Viele Grüße und bis Dienstag,
-
-sendmail_plenum.py
-"""
-
-    msg = EmailMessage()
-    msg['Subject'] = f'Aktuelle Plenumsthemen für Dienstag ({str(d)} 20:00)'
-    msg['From'] = '{{noreply_email_user}}'
-    msg['To'] = 'intern@warpzone.ms'
-    msg['Date'] = formatdate(localtime=True)
-    msg.set_content(mail)
-    
-    server = smtplib.SMTP('{{smtp_host}}', {{smtp_port}})
-    server.starttls()
-    server.login("{{noreply_email_user}}", "{{noreply_email_pass}}")
-    server.send_message(msg)
-    server.quit()
-
+        # Remove empty lines from topics
+        topics = []
+        for extracted in raw[firstline:lastline-1]:
+            if extracted != '':
+                topics.append(extracted)
+
+        # There are no topics - just exit
+        if (len(topics) == 0):
+            return True
+
+        # calculate date of next tuesday
+        d = datetime.date.today()
+        while d.strftime('%a') != 'Tue':
+            d += datetime.timedelta(1)
+
+        # mail template 
+        mail = f"""
+    Liebe Zonies,
+        
+    es gibt wieder Themen die im Plenum besprochen werden sollen. Der nächste
+    Reguläre Termin ist am nächsten Dienstag ({str(d)}) um 20:00.
+    Im Wiki (https://wiki.warpzone.ms/intern:diskussionsthemen#diskussionsthemen_naechstes_plenum_automatische_einladung)
+    stehen folgende Themen:
 
+    {chr(10).join(topics)}
 
+    Die Teilnahme ist natürlich auch remote möglich unter
+    https://jitsi.dorf-post.de/warpzone-plenum möglich.
 
+    Viele Grüße und bis Dienstag,
 
+    sendmail_plenum.py
+    """
 
+        msg = EmailMessage()
+        msg['Subject'] = f'Aktuelle Plenumsthemen für Dienstag ({str(d)} 20:00)'
+        msg['From'] = '{{noreply_email_user}}'
+        msg['To'] = 'intern@warpzone.ms'
+        msg['Date'] = formatdate(localtime=True)
+        msg.set_content(mail)
+        
+        server = smtplib.SMTP('{{smtp_host}}', {{smtp_port}})
+        server.starttls()
+        server.login("{{noreply_email_user}}", "{{noreply_email_pass}}")
+        server.send_message(msg)
+        server.quit()
+    return True
+
+success = False
+try:
+    success = do_work()
+finally:
+    requests.get("{{ healthchecks_url }}" if success else "{{ healthchecks_url }}" + "/fail")

webserver/docker_gitlab/templates/docker-compose.yml

@@ -5,7 +5,7 @@ services:
 
   app:
 
-    image: gitlab/gitlab-ce:15.10.2-ce.0
+    image: gitlab/gitlab-ce:15.11.5-ce.0
     restart: always
     ports:
       - "444:22"

webserver/docker_icinga/tasks/main.yml

@@ -8,6 +8,7 @@
     - { path: "{{ basedir }}/icinga_api_pass",  length: 8 }
     - { path: "{{ basedir }}/mysql_admin_pass",  length: 12 }
     - { path: "{{ basedir }}/mysql_user_pass",  length: 12 }
+    - { path: "{{ basedir }}/matrix_notification_access_token",  length: -1 }
 
 
 - name: pakete installieren

webserver/docker_icinga/templates/Dockerfile

@@ -13,9 +13,10 @@ RUN apt-get update \
     libnet-dns-perl \
     libnet-ip-perl \
     perl \
-    python3-requests \
     python3 \
-    python3-paho-mqtt \
+    python3-pip \
+    python3-requests \
+    virtualenv \
  && apt-get autoremove -y \
  && apt-get clean \
  && rm -rf /tmp/* /var/lib/apt/lists/* /var/cache/debconf/*-old
@@ -48,3 +49,17 @@ RUN cd /opt/ \
  && git clone https://github.com/elberfeld/check_metric_value.git \
  && cd /opt/check_metric_value/ \
  && git checkout b94d3c3e78497a05e3b4520d33421f37e4d77985
+
+# icinga2-matrix-bot - commit from 10.06.2018
+# create a symlink to emulate virtualenv 
+# patch matrix-service-notification.sh
+RUN cd /opt/ \
+ && git clone https://github.com/symptog/icinga2-matrix-bot.git \
+ && cd /opt/icinga2-matrix-bot/ \
+ && git checkout 86d03ed8884950eb2b883a9a94ecf93460b8e0bb \
+ && pip install -r requirements.txt \
+ && mkdir /opt/icinga2-matrix-bot/.venv \
+ && mkdir /opt/icinga2-matrix-bot/.venv/bin \
+ && ln -s /usr/bin/python3 /opt/icinga2-matrix-bot/.venv/bin/python \
+ && sed -i 's/matrix-host-state/matrix-service-state/' icinga2/matrix-service-notification.sh
+ 
\ No newline at end of file

webserver/docker_icinga/templates/etc/icinga/conf.d/commands2.conf

 
+/**
+ * Check MQTT values
+ */
+
 object CheckCommand "check_mqtt" {
   import "plugin-check-command"
 
@@ -33,6 +37,10 @@ object CheckCommand "check_mqtt" {
   }
 }
 
+/**
+ * Check for Mail Blacklisting
+ */
+
 object CheckCommand "check_mail_blacklist" {
   import "plugin-check-command"
 
@@ -45,6 +53,9 @@ object CheckCommand "check_mail_blacklist" {
   }
 }
 
+/**
+ * Check for Prometheus values 
+ */
 
 object CheckCommand "check_metric_value" {
   import "plugin-check-command"
@@ -62,4 +73,56 @@ object CheckCommand "check_metric_value" {
     "-w" = "$metric_warn$"
     "-c" = "$metric_crit$"
   }
-}
\ No newline at end of file
+}
+
+/**
+ * Matrix Notification 
+ */
+
+object NotificationCommand "matrix-host-notification" {
+  import "plugin-notification-command"
+
+  command = [ "/opt/icinga2-matrix-bot/icinga2/matrix-host-notification.sh" ]
+
+  env = {
+    "NOTIFICATIONTYPE" = "$notification.type$"
+    "HOSTALIAS" = "$host.display_name$",
+    "HOSTADDRESS" = "$address$",
+    "HOSTNAME" = "$host.name$",
+    "HOSTSTATE" = "$host.state$",
+    "LONGDATETIME" = "$icinga.long_date_time$",
+    "HOSTOUTPUT" = "$host.output$",
+    "NOTIFICATIONAUTHORNAME" = "$notification.author$",
+    "NOTIFICATIONCOMMENT" = "$notification.comment$",
+    "HOSTDISPLAYNAME" = "$host.display_name$",
+    "ICINGA_WEBURL" = "$notification_icingaweb2url$",
+    "MATRIX_SERVER" = "$user.vars.matrix.server$",
+    "MATRIX_TOKEN" = "$user.vars.matrix.token$",
+    "MATRIX_CHANNEL" = "$user.vars.matrix.channel$",
+  }
+}
+
+object NotificationCommand "matrix-service-notification" {
+  import "plugin-notification-command"
+
+  command = [ "/opt/icinga2-matrix-bot/icinga2/matrix-service-notification.sh" ]
+
+  env = {
+    "NOTIFICATIONTYPE" = "$notification.type$"
+    "SERVICEDESC" = "$service.name$"
+    "HOSTALIAS" = "$host.display_name$",
+    "HOSTNAME" = "$host.name$",
+    "HOSTADDRESS" = "$address$",
+    "SERVICESTATE" = "$service.state$",
+    "LONGDATETIME" = "$icinga.long_date_time$",
+    "SERVICEOUTPUT" = "$service.output$",
+    "NOTIFICATIONAUTHORNAME" = "$notification.author$",
+    "NOTIFICATIONCOMMENT" = "$notification.comment$",
+    "HOSTDISPLAYNAME" = "$host.display_name$",
+    "SERVICEDISPLAYNAME" = "$service.display_name$",
+    "ICINGA_WEBURL" = "$notification_icingaweb2url$",
+    "MATRIX_SERVER" = "$user.vars.matrix.server$",
+    "MATRIX_TOKEN" = "$user.vars.matrix.token$",
+    "MATRIX_CHANNEL" = "$user.vars.matrix.channel$",
+  }
+}

webserver/docker_icinga/templates/etc/icinga/conf.d/notifications.conf

+
+apply Notification "matrix" to Host {
+  import "matrix-host-notification"
+
+  users = [ "matrix" ]
+
+  interval = 4h
+
+  assign where host.address 
+}
+
+
+apply Notification "matrix" to Service {
+  import "matrix-service-notification"
+
+  users = [ "matrix" ]
+
+  interval = 4h
+
+  assign where service.name
+}
+

webserver/docker_icinga/templates/etc/icinga/conf.d/templates.conf

@@ -81,3 +81,30 @@ template Notification "mail-service-notification" {
 
   period = "24x7"
 }
+
+/**
+ * Provides default settings for Matrix.org service notifications.
+ */
+
+template Notification "matrix-host-notification" {
+    command = "matrix-host-notification"
+
+    states = [ Up, Down ]
+    types = [ Problem, Acknowledgement, Custom, FlappingStart, FlappingEnd ]
+    period = "24x7"
+    vars += {
+        notification_icingaweb2url = "https://{{ domain }}/icingaweb2"
+    }
+}
+
+template Notification "matrix-service-notification" {
+    command = "matrix-service-notification"
+
+    states = [ OK, Critical, Unknown ]
+    types = [ Problem, Acknowledgement, Custom, FlappingStart, FlappingEnd ]
+    period = "24x7"
+    vars += {
+        notification_icingaweb2url = "https://{{ domain }}/icingaweb2"
+    }
+}
+

webserver/docker_icinga/templates/etc/icinga/conf.d/users_groups.conf

@@ -3,3 +3,18 @@ object UserGroup "icingaadmins" {
   display_name = "Icinga 2 Admin Group"
 }
 
+object User "matrix" {
+  import "generic-user"
+
+  display_name = "Matrix User"
+  vars.matrix = {
+    token = "{{ matrix_notification_access_token }}"
+    channel = "!iYefxbySFEfFQfUGEK:matrix.warpzone.ms"
+    server = "https://matrix.warpzone.ms"
+  }
+}
+
+object UserGroup "matrix" {
+  display_name = "Matrix Group"
+  assign where user.vars.matrix
+}

webserver/docker_mail/tasks/main.yaml

@@ -69,6 +69,13 @@
     - mailu/overrides/postfix/postfix.cf
   register: config
 
+- name: "set local dns record"
+  become: true
+  blockinfile:
+    path: /etc/hosts
+    create: yes
+    block: |
+      {{ ext_ip4 }} mailserver.warpzone.ms
 
 # - name: deploy LogRotate configs
 #   template:

webserver/docker_mail/templates/docker-compose.yml

@@ -144,6 +144,7 @@ services:
   db:
     image: mariadb:10.5
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    restart: always
     volumes:
       - "{{ basedir }}/db:/var/lib/mysql"
       - "{{ basedir }}/db-init:/docker-entrypoint-initdb.d:ro"

webserver/docker_matrix/templates/docker-compose.yml

@@ -10,7 +10,7 @@ services:
 
   db:
 
-    image: postgres:13.6
+    image: postgres:13.11
     restart: always
     volumes:
       - /srv/matrix/db:/var/lib/postgresql/data
@@ -24,7 +24,7 @@ services:
 
   synapse:
 
-    image: matrixdotorg/synapse:v1.78.0
+    image: matrixdotorg/synapse:v1.83.0
     restart: always
     cpu_count: "1"
     cpuset: "0"