icinga monitoring

common/docker/templates/daemon.json

@@ -4,7 +4,9 @@
     "log-opts": {
       "max-size": "128m",
       "max-file": "5"
-    }
+    },
+    "metrics-addr": "{{int_ip4}}:9323",
+    "experimental": true
 
 }
 

common/docker_dockerstats/tasks/main.yml

+---
+
+
+- name: create folder struct for {{ servicename }}
+  file: 
+    path: "{{ item }}"
+    state: "directory"
+  with_items:
+    - "{{ basedir }}"
+
+
+- name: Konfig-Dateien erstellen
+  template:
+    src: "{{ item }}"
+    dest: "{{ basedir }}/{{ item }}"
+  with_items:
+    - Dockerfile
+    - docker-compose.yml
+
+
+- name: start {{ servicename }} docker
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: present

common/docker_dockerstats/templates/Dockerfile

+FROM node:14-alpine
+
+RUN apk update \
+ && apk upgrade \
+ && apk add --no-cache git
+
+RUN mkdir -p /usr/src/app \
+  && cd /usr/src/app \
+  && git clone https://github.com/elberfeld/docker_stats_exporter.git \
+  && cd /usr/src/app/docker_stats_exporter \
+  && git checkout 2020.07.30.1 \
+  && npm install
+
+WORKDIR /usr/src/app/docker_stats_exporter
+
+EXPOSE 9487
+ENV DOCKERSTATS_PORT=9487 DOCKERSTATS_INTERVAL=15 DEBUG=0
+
+ENTRYPOINT [ "npm", "start" ]

common/docker_dockerstats/templates/docker-compose.yml

+version: "3"
+
+services:
+
+  app:
+
+    build: .
+    restart: always
+    ports:
+      - "{{ int_ip4 }}:9487:9487" 
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /usr/bin/docker:/usr/bin/docker
+

common/prometheus-node/tasks/main.yml

 ---
-
-
-- name: stop prometheus-node-exporter
-  service: name=prometheus-node-exporter state=stopped
-
-# Pakete deinstallieren
-- name: pakete deinstallieren
+# Pakete installieren
+- name: pakete installieren
   apt:
     pkg: "{{ item }}"
-    update_cache: no
-    state: absent
+    update_cache: yes
+    state: installed
   with_items:
     - prometheus-node-exporter
 
+- name: remove old directories 
+  file: 
+    path: "{{ item }}"
+    state: "absent"
+  with_items:
+    - /srv/prometheus-node-exporter 
+
+- name: Configure Node-Exporter 
+  lineinfile:
+    path: /etc/default/prometheus-node-exporter
+    regexp: '^ARGS='
+    line: ARGS="--web.listen-address={{int_ip4}}:9100"
+
+- name: reload systemd and enable service
+  command: systemctl enable prometheus-node-exporter
+
+- name: restart prometheus-node-exporter
+  service: 
+    name: prometheus-node-exporter 
+    state: restarted
+

group_vars/prod

@@ -17,7 +17,8 @@ ldap_domain: warpzone.ms
 ldap_base_dn: dc=warpzone,dc=ms
 ldap_admin_bind_dn: cn=admin,dc=warpzone,dc=ms
 ldap_readonly_bind_dn: cn=readonly,dc=warpzone,dc=ms
-
+ldap_group_dn: ou=groups,dc=warpzone,dc=ms
+ldap_group_active_dn: cn=active,ou=groups,dc=warpzone,dc=ms
 
 # SMTP Settings 
 smtp_domain: warpzone.ms
@@ -25,6 +26,10 @@ smtp_host: mailserver.warpzone.ms
 smtp_port: 587 
 noreply_email_user: noreply@warpzone.ms
 
+# Globale Domains
+global_domains:
+  warpzonems:
+    domain: warpzone.ms
 
 # Globale Mail konfiguration 
 mail_domains:
@@ -32,6 +37,38 @@ mail_domains:
     maildomain: warpzone.ms 
     mxserver: mailserver.warpzone.ms
     mxhostname: webserver
+    spf: v=spf1 mx ~all
+    dmarc: v=DMARC1; p=none;
+    dkim:
+      - { selector: "dkim", value: "v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+ZvoSoa2LwBbzQMD9laVy8hUGbvhe1LkL/6SIk3Ks8GfiT7p+hdlbcvo+noBR4gvbmSWwn3yBxOnGCtSH+iP0q7HHrmeEXJqGkLK25zZh1EO8bZqIHi2NX/LnN7dJTO8C27CRLME+YtWdrDaerIWXsHk7U+qD1ZuM5Q+FgAzsQ5uxQVlD6sO3IU" }
+  member_warpzonems:
+    maildomain: member.warpzone.ms 
+    mxserver: mailserver.warpzone.ms
+    mxhostname: webserver
+    spf: v=spf1 mx ~all
+    dmarc: v=DMARC1; p=none;
+    dkim:
+      - { selector: "dkim", value: "v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8lZDykC3gbxSHMwTNO7QrDytlO9Sg66nEXpIv1/GqQrj3T1i3tTn05XxpJbRXUMuooaP6xZqt2OR3f/Wex6d4WwHH4Z1YuvyKDUWewynGZ3Ge+Vca8T0LBdDw7DZWtkXv94SHPWLyPWuuBXQs2nAgrMn3rtlwKovEsOqg85mFNb1EVm9Rgj9TB2" }
+  lists_warpzonems:
+    maildomain: lists.warpzone.ms 
+    mxserver: mailserver.warpzone.ms
+    mxhostname: webserver
+    spf: v=spf1 mx ~all
+    dmarc: v=DMARC1; p=none;
+#  chaostreffmuensterde:
+#    maildomain: chaostreff-muenster.de 
+#    mxserver: mailserver.warpzone.ms
+#    mxhostname: webserver
+#    spf: v=spf1 mx ~all
+#    dmarc: v=DMARC1; p=none;
+#    dkim:
+#      - { selector: "dkim", value: "v=DKIM1;k=rsa;t=s;s=email;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz/OBnxYygjhKeZVyvhDAO1/O1XwyYEhQx3bW/rO/Wmp8ZzP/eQh3dljDEibj1KsfdUhfgTIU8CnTKLayb8B07MMzhBklpg8WUV2LrDmpndfhixizjaxzwBj/dhtiZE7e4BwhOPOmdBQ0cCIvNhMcQcCa1RgCpX/g5Ii0AtQ2zCPMTSOW5YWn+VY" }
+#  lists_chaostreffmuensterde:
+#    maildomain: lists.chaostreff-muenster.de 
+#    mxserver: mailserver.warpzone.ms
+#    mxhostname: webserver
+#    spf: v=spf1 mx ~all
+#    dmarc: v=DMARC1; p=none;
 
 
 # Zentrale InfluxDb für Systemmonitoring  
@@ -53,3 +90,14 @@ matrix:
   domain: matrix.warpzone.ms
   public_url: https://matrix.warpzone.ms
   identity_server: https://matrix.warpzone.ms
+
+# Monitoring 
+monitoring:
+  internal_ldap_servers:
+    - webserver
+    - verwaltung
+    - warpsrvint
+  external_dns_servers:
+    - { ip: "8.8.8.8",      name: "Google" }
+    - { ip: "9.9.9.9",      name: "Quad9" }
+    - { ip: "46.182.19.48", name: "Digitalcourage" }

host_vars/verwaltung

@@ -58,6 +58,31 @@ vorstandteam:
   - "h3rb3rn"
   - "mowoe"
 
+# Monitoring aktivieren 
+alert:  
+  load: 
+    warn: 8
+    crit: 16
+  containers:
+    - { name: "dockerstats_app_1" }
+    - { name: "gitea_app_1" }
+    - { name: "gitea_db_1" }
+    - { name: "jameica-vnc_ldap_auth_1" }
+    - { name: "jameica-vnc_nginx_1" }
+    - { name: "jameica-vnc_vnc_1" }
+    - { name: "ldap_openldap_1" }
+    - { name: "ldap_phpldapadmin_1" }
+    - { name: "ldap_syncreplexporter_1" }
+    - { name: "mysql_app_1_aa1ef2868e9c" }
+    - { name: "nextcloud_app_1" }
+    - { name: "nextcloud_elasticsearch_1" }
+    - { name: "nextcloud_mysql_1" }
+    - { name: "nextcloud_redis_1" }
+    - { name: "traefik_app_1" }
+  disks: 
+    - { mountpoint: "/", warn: "5 GB", crit: "1 GB" }
+    - { mountpoint: "/srv", warn: "5 GB", crit: "1 GB" }
+
 
 # Definition von Borgbackup Repositories 
 borgbackup_repos:
@@ -85,6 +110,13 @@ borgbackup_repos:
     #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
     # directories:
 
+    # Monitoring
+    alert: true
+    warning_age: 26 
+    critical_age: 50
+    warning_count: 10
+    critical_count: 5
+
   borgbase: 
 
     # URL des Repos   
@@ -108,6 +140,12 @@ borgbackup_repos:
     #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
     # directories:
 
+    # Monitoring
+    alert: true
+    warning_age: 26 
+    critical_age: 50
+    warning_count: 10
+    critical_count: 5
 
 # Definition der Verzeichnisse, die in allen Borgbackup Repos gesichert werden sollen 
 borgbackup_directories:

host_vars/warpsrvint

@@ -41,37 +41,40 @@ administratorenteam:
   - "dray"
   - "sandhome"
 
+# Monitoring aktivieren 
+alert:  
+  load: 
+    warn: 2
+    crit: 4
+  containers:
+    - { name: "dockerstats_app_1" }
+    - { name: "influx_sysmon_1" }
+    - { name: "grafana_app_1" }
+    - { name: "unifi_app_1" }
+    - { name: "ldap_phpldapadmin_1" }
+    - { name: "matestatdb_db_1" }
+    - { name: "l4z0r_db_1" }
+    - { name: "warpinfratest_app_1" }
+    - { name: "warpinfratest_db_1" }
+    - { name: "nodered-app" }
+    - { name: "prometheus-alert" }
+    - { name: "prometheus-statsd-exporter" }
+    - { name: "prometheus-snmp-exporter" }
+    - { name: "prometheus-blackbox-exporter" }
+    - { name: "mqtt-service" }
+    - { name: "warpinfra-db" }
+    - { name: "warpinfra-app" }
+  disks: 
+    - { mountpoint: "/", warn: "5 GB", crit: "1 GB" }
+
 
 # Definition von Borgbackup Repositories 
 borgbackup_repos:
   
-  warpsrvext: 
-
-    # URL des Repos   
-    repo: "ssh://warpzone@217.79.181.126:22/data/warpzone/warpsrvint"
-    
-    # Repo-spezifische Optionen zum Aufruf von Borgbackup
-    # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
-    options: ""
-
-    # Compression Options, z,b. "zlib,5, "zstd,5"
-    compression: "zlib,5"
-
-    # Prune Optionen 
-    prune: "--keep-within=2d --keep-daily=7 --keep-weekly=4 --keep-monthly=6"
-    
-    # Backup Schedule 
-    weekday: "*"
-    hour: "*/4"
-    minute: "10"
-
-    #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
-    # directories:
-
   borgbase: 
 
     # URL des Repos   
-    repo: "w3299kpl@w3299kpl.repo.borgbase.com:repo"
+    repo: "u127404b@u127404b.repo.borgbase.com:repo"
     
     # Repo-spezifische Optionen zum Aufruf von Borgbackup
     # z.B. bei Sicherungen zu rsync.net ist --remote-path=borg1 erforderlich
@@ -91,6 +94,13 @@ borgbackup_repos:
     #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
     # directories:
 
+    # Monitoring
+    alert: true
+    warning_age: 10
+    critical_age: 20
+    warning_count: 10
+    critical_count: 5
+
 
 # Definition der Verzeichnisse, die in allen Borgbackup Repos gesichert werden sollen 
 borgbackup_directories:

host_vars/webserver

@@ -21,7 +21,6 @@ ext_ip4: 89.163.231.226
 ext_ip6: 2001:4ba0:ffff:7c::2
 int_ip4: 10.42.1.1
 
-
 # Art des Hosts: physical, vm, docker 
 host_type: "vm"
 
@@ -32,7 +31,7 @@ webserver_ssl: true
 webserver_domains: 
   - "warpzone.ms"
   - "api.warpzone.ms"
-  - "auth.warpzone.ms"
+#  - "auth.warpzone.ms"
   - "autodiscover.warpzone.ms"
   - "autoconfig.warpzone.ms"
   - "gitlab.warpzone.ms"
@@ -42,11 +41,7 @@ webserver_domains:
   - "mailserver.warpzone.ms"
   - "muc.jabber.warpzone.ms"
   - "proxy.jabber.warpzone.ms"
-  - "jabber-test.warpzone.ms"
-  - "muc.jabber-test.warpzone.ms"
-  - "proxy.jabber-test.warpzone.ms"
   - "ldap.warpzone.ms"
-  - "mattermost.warpzone.ms"
   - "md.warpzone.ms"
   - "wiki.warpzone.ms"
   - "www.warpzone.ms"
@@ -60,6 +55,60 @@ openvpn_server:
 administratorenteam:
   - "void"
   - "sandhome"
+
+
+# Monitoring aktivieren 
+alert:  
+  load: 
+    warn: 8
+    crit: 16
+  containers:
+    - { name: "autodiscover_warpzonems_1" }
+    - { name: "dockerstats_app_1" }
+    - { name: "dokuwiki_app_1" }
+    - { name: "gitlab_app_1" }
+    - { name: "hackmd_app_1" }
+    - { name: "hackmd_db_1" }
+    - { name: "icinga_app_1" }
+    - { name: "icinga_db_1" }
+    - { name: "icinga_graphite_1" }
+    - { name: "jabber_app_1" }
+    - { name: "ldap_openldap_1" }
+    - { name: "ldap_phpldapadmin_1" }
+    - { name: "ldap_syncreplexporter_1" }
+    - { name: "mail_dovecot-mailcow_1" }
+    - { name: "mail_dockerapi-mailcow_1" }
+    - { name: "mail_ipv6nat-mailcow_1" }
+    - { name: "mail_mailman-core" }
+    - { name: "mail_mailman-db" }
+    - { name: "mail_mailman-nginx" }
+    - { name: "mail_mailman-web" }
+    - { name: "mail_memcached-mailcow_1" }
+    - { name: "mail_mysql-mailcow_1" }
+    - { name: "mail_netfilter-mailcow_1" }
+    - { name: "mail_nginx-mailcow_1" }
+    - { name: "mail_olefy-mailcow_1" }
+    - { name: "mail_postfix-mailcow_1" }
+    - { name: "mail_php-fpm-mailcow_1" }
+    - { name: "mail_redis-mailcow_1" }
+    - { name: "mail_rspamd-mailcow_1" }
+    - { name: "mail_traefik-certdumper_1" }    
+    - { name: "mail_unbound-mailcow_1" }
+    - { name: "mail_watchdog-mailcow_1" }
+    - { name: "matterbridge_cw_1" }
+    - { name: "matterbridge_wz_1" }
+    - { name: "matrix_ma1sd_1" }
+    - { name: "matrix_db_1" }
+    - { name: "matrix_synapse_1" }
+    - { name: "traefik_app_1" }
+    - { name: "warpapi_app_1" }
+    - { name: "warpinfra_app_1" }
+    - { name: "warpinfra_db_1" }
+    - { name: "wordpress_app_1" }
+    - { name: "wordpress_db_1" }
+  disks: 
+    - { mountpoint: "/", warn: "5 GB", crit: "1 GB" }
+    - { mountpoint: "/var/lib/docker", warn: "1 GB", crit: "500 MB" }
   
 
 # Definition von Borgbackup Repositories 
@@ -88,6 +137,13 @@ borgbackup_repos:
     #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
     # directories:
 
+    # Monitoring
+    alert: true
+    warning_age: 26 
+    critical_age: 50
+    warning_count: 10
+    critical_count: 5
+
   borgbase: 
 
     # URL des Repos   
@@ -111,6 +167,13 @@ borgbackup_repos:
     #  Zusätzliche Verzeichnisse, die nur in diesem Backup gesichtert werden sollen 
     # directories:
 
+    # Monitoring
+    alert: true
+    warning_age: 26 
+    critical_age: 50
+    warning_count: 10
+    critical_count: 5
+
 
 # Definition der Verzeichnisse, die in allen Borgbackup Repos gesichert werden sollen 
 borgbackup_directories:

site.yml

@@ -26,7 +26,12 @@
     - { role: common/borgbackup, tags: borgbackup }
     - { role: common/borgserver, tags: borgserver }
     - { role: common/docker, tags: docker }
-    - { role: common/telegraf, tags: telegraf }
+    - { role: common/prometheus-node, tags: prometheus-node }
+    - { 
+        role: common/docker_dockerstats, tags: dockerstats, 
+        servicename: dockerstats, 
+        basedir: /srv/dockerstats 
+      }
     - { role: common/docker_ldap, tags: ldap }
     - { role: common/nginx, tags: nginx }
     - { role: warpsrvint/docker_grafana, tags: grafana }
@@ -49,6 +54,12 @@
     - { role: common/borgbackup, tags: borgbackup }
     - { role: common/docker, tags: docker }
     - { role: common/openvpn, tags: openvpn }
+    - { role: common/prometheus-node, tags: prometheus-node }
+    - { 
+        role: common/docker_dockerstats, tags: dockerstats, 
+        servicename: dockerstats, 
+        basedir: /srv/dockerstats 
+      }
     - { 
         role: common/docker_ldap, tags: ldap,
         servicename: "ldap",
@@ -75,6 +86,14 @@
         servicename: "gitlab",
         domain: "gitlab.warpzone.ms"
       }
+    - { 
+        role: webserver/docker_icinga, tags: icinga, 
+        servicename: icinga, 
+        basedir: /srv/icinga, 
+        domain: icinga.warpzone.ms,
+        api_port: 5665,
+        mysql_port: 33306, 
+      }
     - { 
         role: webserver/docker_hackmd, tags: hackmd,
         servicename: "hackmd",
@@ -89,7 +108,8 @@
         role: webserver/docker_mail, tags: mail 
       }
     - { 
-        role: webserver/docker_matterbridge, tags: matterbridge 
+        role: webserver/docker_matterbridge, tags: matterbridge,
+        domain: "www.warpzone.ms" 
       }
     - { 
         role: webserver/docker_matrix, tags: matrix,
@@ -123,6 +143,12 @@
     - { role: common/borgbackup, tags: borgbackup }
     - { role: common/docker, tags: docker }
     - { role: common/openvpn, tags: openvpn }
+    - { role: common/prometheus-node, tags: prometheus-node }
+    - { 
+        role: common/docker_dockerstats, tags: dockerstats, 
+        servicename: dockerstats, 
+        basedir: /srv/dockerstats 
+      }
     - { 
         role: common/docker_ldap, tags: ldap,
         servicename: "ldap",

webserver/docker_icinga/handlers/main.yml

+---
+
+- name: restart icinga docker
+  docker_compose:
+    project_src: /srv/icinga/
+    state: present
+    restarted: yes

webserver/docker_icinga/tasks/main.yml

+---
+
+- include: ../functions/get_secret.yml
+  with_items:
+    - { path: /srv/ldap/secret/ldap_readonly_pass, length: -1 }
+    - { path: "{{ basedir }}/icinga_admin_pass",  length: 12 }
+    - { path: "{{ basedir }}/icinga_api_user",  length: 8 }
+    - { path: "{{ basedir }}/icinga_api_pass",  length: 8 }
+    - { path: "{{ basedir }}/mysql_admin_pass",  length: 12 }
+    - { path: "{{ basedir }}/mysql_user_pass",  length: 12 }
+
+
+- name: pakete installieren
+  apt:
+    update_cache: no
+    state: present
+    name: 
+      - logrotate
+
+- name: icinga LogRotate config erstellen 
+  template: 
+    src: logrotate 
+    dest: /etc/logrotate.d/icinga
+
+
+- name: "create folder struct for {{ servicename }}"
+  file: 
+    path: "{{ item }}"
+    state: "directory"
+  with_items:
+    - "{{ basedir }}"
+    - "{{ basedir }}/data/"
+    - "{{ basedir }}/etc/"
+    - "{{ basedir }}/log/"
+    - "{{ basedir }}/db/"
+    - "{{ basedir }}/graphite-conf/"
+    - "{{ basedir }}/graphite-storage/"
+
+
+- name: Konfig-Dateien erstellen (base,graphite)
+  template:
+    src: "{{ item }}"
+    dest: "{{ basedir }}/{{ item }}"
+  with_items:
+    - Dockerfile
+    - docker-compose.yml
+    - check_rbl_helper.sh
+    - notify_by_pushover.sh
+    - etc/locale.gen
+    - graphite-conf/storage-schemas.conf
+  notify: restart icinga docker
+
+
+- stat:
+    path: "{{ basedir }}/etc/icingaweb2/CONFIGURED"
+  register: configured
+
+- name: "start {{ servicename }} docker (init)"
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: present
+  when: configured.stat.exists == False
+
+- name: "wait for {{ servicename }} docker (init)"
+  wait_for:
+    path: "{{ basedir }}/etc/icingaweb2/CONFIGURED"
+  when: configured.stat.exists == False
+
+- name: "stop {{ servicename }} docker (init)"
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: absent
+  when: configured.stat.exists == False
+
+
+- name: Script Helper erstellen
+  template: 
+    src: "{{ item }}" 
+    dest: "{{ basedir }}/{{ item }}"
+    mode: u+x
+  with_items:
+    - debuglog_enable.sh
+    - debuglog_disable.sh
+
+
+- name: Konfig-Dateien erstellen (icinga,icingaweb2)
+  template:
+    src: "{{ item }}"
+    dest: "{{ basedir }}/{{ item }}"
+  with_items:
+    - etc/icinga/conf.d/api-users.conf
+    - etc/icinga/conf.d/commands2.conf
+    - etc/icinga/conf.d/groups.conf
+    - etc/icinga/conf.d/hosts_manual.conf
+    - etc/icinga/conf.d/hosts.conf
+#    - etc/icinga/conf.d/notifications_pushover.conf
+    - etc/icinga/conf.d/notifications.conf
+    - etc/icinga/conf.d/services_backup.conf
+    - etc/icinga/conf.d/services_container.conf
+    - etc/icinga/conf.d/services_domains.conf
+    - etc/icinga/conf.d/services_exporters.conf
+#    - etc/icinga/conf.d/services_ldap.conf
+    - etc/icinga/conf.d/services_mail.conf
+    - etc/icinga/conf.d/services_manual.conf
+#    - etc/icinga/conf.d/services_mqttsensors.conf
+    - etc/icinga/conf.d/services_system.conf
+    - etc/icinga/conf.d/services.conf
+    - etc/icinga/conf.d/templates.conf
+    - etc/icinga/conf.d/users_groups.conf
+    - etc/icinga/conf.d/users_sample.conf
+    - etc/icingaweb2/authentication.ini
+    - etc/icingaweb2/groups.ini
+    - etc/icingaweb2/resources.ini
+    - etc/icingaweb2/roles.ini
+  notify: restart icinga docker
+
+  
+- name: "start {{ servicename }} docker"
+  docker_compose:
+    project_src: "{{ basedir }}"
+    state: present

webserver/docker_icinga/templates/Dockerfile

+FROM jordan/icinga2:2.12.1
+
+# Install additional Packages (Backports needed for Golang > 1.13)
+RUN apt-get update \
+ && apt-get install -y -q --no-install-recommends -t buster-backports \
+    curl \ 
+    dnsutils \
+    git \
+    golang \
+    jq \
+    libdata-validate-domain-perl \
+    libdata-validate-ip-perl \
+    libmonitoring-plugin-perl \
+    libnet-dns-perl \
+    libnet-ip-perl \
+    perl \
+    python-requests \
+    python3 \
+    python3-paho-mqtt \
+ && apt-get autoremove -y \
+ && apt-get clean \
+ && rm -rf /tmp/* /var/lib/apt/lists/* /var/cache/debconf/*-old
+
+# Helper Scripe  
+COPY check_rbl_helper.sh /opt 
+COPY notify_by_pushover.sh /opt 
+RUN chmod +x /opt/*.sh
+
+# check_mqtt
+RUN cd /opt/ && git clone https://github.com/jpmens/check-mqtt.git
+
+# check_rbl
+RUN cd /opt/ && git clone https://github.com/matteocorti/check_rbl.git
+
+# check_json
+RUN cd /opt/ && git clone https://github.com/asymworks/check_json.git
+
+# prom2json (go 1.13 min reqired)
+RUN cd /opt/ && GOPATH=/opt/ go get github.com/prometheus/prom2json/cmd/prom2json
+
+# check_metric_value
+RUN cd /opt/ && git clone https://github.com/elberfeld/check_metric_value.git

webserver/docker_icinga/templates/check_rbl_helper.sh

+#!/bin/bash
+/usr/bin/perl /opt/check_rbl/check_rbl --extra-opts=rbl@/opt/check_rbl/check_rbl.ini $@

webserver/docker_icinga/templates/debuglog_disable.sh

+#!/bin/sh 
+
+cd /srv/icinga 
+docker-compose exec app icinga2 feature disable debuglog
+docker-compose restart 
+rm log/icinga2/debug.log

webserver/docker_icinga/templates/debuglog_enable.sh

+#!/bin/sh 
+
+cd /srv/icinga 
+docker-compose exec app icinga2 feature enable debuglog
+docker-compose restart 
+tail -f log/icinga2/debug.log

webserver/docker_icinga/templates/docker-compose.yml

+
+
+version: "2"
+
+services:
+  
+  app:
+
+    build: .
+    restart: always
+    mem_limit: 512m
+    hostname: "{{ domain }}"
+    ports:
+      - "0.0.0.0:{{ api_port }}:5665"
+    volumes:
+      - "{{ basedir }}/data:/var/lib/icinga2"
+      - "{{ basedir }}/etc/locale.gen:/etc/locale.gen"
+      - "{{ basedir }}/etc/icinga:/etc/icinga2"
+      - "{{ basedir }}/etc/icingaweb2:/etc/icingaweb2"
+      - "{{ basedir }}/log/apache2:/var/log/apache2"
+      - "{{ basedir }}/log/icinga2:/var/log/icinga2"
+      - "{{ basedir }}/log/icingaweb2:/var/log/icingaweb2"
+    depends_on:
+      - db
+      - graphite
+    environment:
+      TZ: "Europe/Berlin"
+      APACHE2_HTTP: BOTH
+      MYSQL_HOST: db
+      MYSQL_ROOT_USER: "root"
+      MYSQL_ROOT_PASSWORD: "{{ mysql_admin_pass }}"
+      MYSQL_PASSWORD: "{{ mysql_user_pass }}"
+      MYSQL_DATABASE: icinga
+      MYSQL_USER: icinga
+      DEFAULT_MYSQL_HOST: db
+      DEFAULT_MYSQL_USER: icinga
+      DEFAULT_MYSQL_PASS: "{{ mysql_user_pass }}"
+      ICINGAWEB2_ADMIN_PASS: "{{ icinga_admin_pass }}"
+      ICINGA2_FEATURE_GRAPHITE: 1
+      ICINGA2_FEATURE_GRAPHITE_HOST: graphite
+      ICINGA2_FEATURE_GRAPHITE_PORT: 2003
+      ICINGA2_FEATURE_DIRECTOR: 0
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.{{ servicename }}.rule=Host(`{{ domain }}`)
+      - traefik.http.routers.{{ servicename }}.entrypoints=websecure
+      - traefik.http.services.{{ servicename }}.loadbalancer.server.port=80      
+    networks:
+      - default
+      - web
+
+  db:
+
+    image: mariadb:10.5.6
+    restart: always
+    mem_limit: 256m
+    ports:
+      - "{{ int_ip4 }}:{{mysql_port}}:3306"
+    volumes:
+      - "{{ basedir }}/db:/var/lib/mysql"
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ mysql_admin_pass }}"
+      MYSQL_PASSWORD: "{{ mysql_user_pass }}"
+      MYSQL_DATABASE: icinga
+      MYSQL_USER: icinga
+    networks:
+      - default
+  
+  graphite:
+    
+    image: graphiteapp/graphite-statsd:1.1.7-6
+    restart: always
+    mem_limit: 256m
+    volumes:
+      - "{{ basedir }}/graphite-conf/storage-schemas.conf:/opt/graphite/conf/storage-schemas.conf"
+      - "{{ basedir }}/graphite-storage:/opt/graphite/storage"
+    environment:
+      GRAPHITE_TIME_ZONE: "Europe/Berlin"
+      GRAPHITE_DATE_FORMAT: "%d.%m.%y"
+      GRAPHITE_LOG_FILE_INFO: "-"
+      GRAPHITE_LOG_FILE_EXCEPTION: "-"
+      GRAPHITE_LOG_FILE_CACHE: "-"
+      GRAPHITE_LOG_FILE_RENDERING: "-"
+    networks:
+      - default
+  
+networks:
+  web:
+    external: true    
\ No newline at end of file

webserver/docker_icinga/templates/etc/icinga/conf.d/api-users.conf

+/**
+ * The ApiUser objects are used for authentication against the API.
+ */
+object ApiUser "{{icinga_api_user}}" {
+  password = "{{icinga_api_pass}}"
+  permissions = [ "*" ]
+}

webserver/docker_icinga/templates/etc/icinga/conf.d/commands2.conf

+
+object CheckCommand "check_mqtt" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check-mqtt/check-mqtt.py" ] 
+
+  arguments = {
+    "-H" = "$mqtt_host$"
+    "-u" = "$mqtt_user$"
+    "-p" = "$mqtt_password$"
+    "-P" = "$mqtt_port$"
+    "-a" = "$mqtt_cafile$"
+    "-C" = "$mqtt_certfile$"
+    "-k" = "$mqtt_keyfile$"
+    "-t" = "$mqtt_topic$"
+    "-m" = {
+      set_if = "$mqtt_max$"
+      value = "$mqtt_max$"
+    }
+
+    "-l" = "$mqtt_payload$"
+    "-v" = "$mqtt_value$"
+    "-o" = "$mqtt_operator$"
+
+    "-r" = {
+      set_if = "$mqtt_readonly$"
+      description = "Don't write."
+    }
+    "-n" = {
+      set_if = "$mqtt_insecure$"
+      description = "suppress TLS hostname check"
+    }
+  }
+}
+
+object CheckCommand "check_mail_blacklist" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check_rbl_helper.sh" ] 
+
+  arguments = {
+    "-H" = "$rbl_host$"
+    "-c" = "$rbl_critical$"
+    "-w" = "$rbl_warning$"
+  }
+}
+
+
+object CheckCommand "check_metric_value" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check_metric_value/check_metric_value.py" ] 
+
+  arguments = {
+    "-P" = "/opt/bin/prom2json"
+    "-U" = "$metric_url$"
+    "-M" = "$metric_name$"
+    "-n" = "$metric_labelname$"
+    "-v" = "$metric_labelvalue$"
+    "-o" = "$metric_operator$"
+    "-u" = "$metric_unit$"
+    "-w" = "$metric_warn$"
+    "-c" = "$metric_crit$"
+  }
+}
\ No newline at end of file

webserver/docker_icinga/templates/etc/icinga/conf.d/groups.conf

+/**
+ * Host groups 
+ */
+
+object HostGroup "linux-servers" {
+  display_name = "Linux Servers"
+
+  assign where host.vars.os == "Linux"
+}
+
+/*
+object HostGroup "windows-servers" {
+  display_name = "Windows Servers"
+
+  assign where host.vars.os == "Windows"
+}
+*/
+
+object HostGroup "network" {
+  display_name = "Network Devices"
+}
+
+object HostGroup "other" {
+  display_name = "Other Devices"
+}
+
+/**
+ * Service groups by check command
+ */
+
+object ServiceGroup "ping" {
+  display_name = "Ping Checks"
+
+  assign where match("ping*", service.name)
+}
+
+object ServiceGroup "http" {
+  display_name = "HTTP Checks"
+
+  assign where match("http*", service.check_command)
+}
+
+object ServiceGroup "dns" {
+  display_name = "DNS Checks"
+
+  assign where match("dig*", service.check_command)
+}
+
+object ServiceGroup "mqtt" {
+  display_name = "MQTT Checks"
+
+  assign where match("check_mqtt*", service.check_command)
+}
+
+/**
+ * Service Goups assigned in Services
+ */
+
+object ServiceGroup "backup" {
+  display_name = "Backup Checks"
+}
+
+object ServiceGroup "container" {
+  display_name = "Docker Container Checks"
+}
+
+object ServiceGroup "ldap" {
+  display_name = "LDAP Checks"
+}
+
+object ServiceGroup "certificate" {
+  display_name = "Certificate Checks"
+}
+
+object ServiceGroup "mail" {
+  display_name = "Mail Checks"
+}
+
+object ServiceGroup "exporter" {
+  display_name = "Metrics Exporter Checks"
+}