icinga test (startet noch nicht richtig)

group_vars/test

@@ -18,7 +18,7 @@ mail_domains:
     spf: "v=spf1 mx a:mailserver.test-warpzone.de ip4:{{ hostvars['test-warpzone-de'].ext_ip4 }} ip6:{{ hostvars['test-warpzone-de'].ext_ip6 }} -all"
     dmarc: "v=DMARC1; p=none;"
     dkim:
-      - { selector: "dkim", value: "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqlvNCdae1bHGiuBrbXLwbtlEeQySngzG5wa7rG9O7eFFBnEKsrk9yOexRt1N5rOotRwL+Zy/9So8nylUFggP8nXlCgfUmEDPfNaWfzAeHUaPCTpUpbEZhOWr0vGxAyWeJ2p2eOAFK2PUU+KVqF7L3Zhb3yQxyYPKhKD4uxwgqH/Y2UPNP0SaJ7fOtZpW" "1cGiIVp2aVGiE5w1AbI3kDfLpGuh8g0AzBknVX4z8wb+f5wdZiX/3/iebv8LVxOpu6DRMt48D9PN9hRQywDVLPNko03rSu5MHoz3ilJC7lkFg7DRUssFT4JHeyrxoOu7FhZUc8BKjeQ3W2mrsGd6Y48ffQIDAQAB" }
+      - { selector: "dkim", value: "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqlvNCdae1bHGiuBrbXLwbtlEeQySngzG5wa7rG9O7eFFBnEKsrk9yOexRt1N5rOotRwL+Zy/9So8nylUFggP8nXlCgfUmEDPfNaWfzAeHUaPCTpUpbEZhOWr0vGxAyWeJ2p2eOAFK2PUU+KVqF7L3Zhb3yQxyYPKhKD4uxwgqH/Y2UPNP0SaJ7fOtZpW\" \"1cGiIVp2aVGiE5w1AbI3kDfLpGuh8g0AzBknVX4z8wb+f5wdZiX/3/iebv8LVxOpu6DRMt48D9PN9hRQywDVLPNko03rSu5MHoz3ilJC7lkFg7DRUssFT4JHeyrxoOu7FhZUc8BKjeQ3W2mrsGd6Y48ffQIDAQAB" }
   # lists_warpzonems:
   #   maildomain: "lists.test-warpzone.de" 
   #   mxserver: "mailserver.test-warpzone.de"
@@ -33,3 +33,13 @@ monitoring:
     - { ip: "1.1.1.1", name: "Cloudflare" }
     - { ip: "8.8.8.8", name: "Google" }
     - { ip: "9.9.9.9", name: "Quad9" }
+
+
+
+# Globale OAuth Server Settings
+oauth_global:
+  authorize_url: https://uffd.test-warpzone.de/oauth2/authorize
+  token_url: https://uffd.test-warpzone.de/oauth2/token
+  userinfo_url: https://uffd.test-warpzone.de/oauth2/userinfo
+  logout_url: https://uffd.test-warpzone.de/logout
+  metrics_url: https://uffd.test-warpzone.de/metrics
\ No newline at end of file

site.yml

@@ -51,19 +51,28 @@
     #     domain: "www.test-warpzone.de"
     #   }
     - { 
-        role: testserver/docker_mail, tags: mail, 
+        role: testserver/docker_mail, tags: test_mail, 
         servicename: mail,
-        basedir: /srv/mail,
+        basedir: "/srv/{{ servicename }}",
         domain: "test-warpzone.de",
         mailserver: "mailserver.test-warpzone.de",
         listserver: "listserver.test-warpzone.de"
       }
     - { 
-        role: testserver/docker_uffd, tags: uffd, 
+        role: testserver/docker_uffd, tags: test_uffd, 
         servicename: uffd, 
-        basedir: /srv/uffd, 
+        basedir: "/srv/{{ servicename }}",
         domain: "uffd.test-warpzone.de",
       }
+    - { 
+        role: testserver/docker_icinga, tags: test_icinga,
+        servicename: icinga,
+        basedir: "/srv/{{ servicename }}",
+        domain: "icinga.test-warpzone.de",
+        groupname: test,
+        api_port: 5665,
+        mysql_port: 33306
+    }
 
 
 
@@ -242,6 +251,7 @@
         servicename: icinga, 
         basedir: /srv/icinga, 
         domain: icinga.warpzone.ms,
+        groupname: prod,
         api_port: 5665,
         mysql_port: 33306
       }

testserver/docker_icinga/handlers/main.yml

+---
+
+- name: restart icinga docker
+  docker_compose:
+    project_src: /srv/icinga/
+    state: present
+    restarted: yes

testserver/docker_icinga/tasks/main.yml

+---
+
+- include_tasks: ../functions/get_secret.yml
+  with_items:
+    - { path: "{{ basedir }}/secrets/forward_auth_secret", length: 64 }
+    - { path: "{{ basedir }}/secrets/oauth_client_secret", length: 64 }
+    - { path: "{{ basedir }}/secrets/icinga_admin_pass",   length: 12 }
+    - { path: "{{ basedir }}/secrets/icinga_api_user",     length: 8 }
+    - { path: "{{ basedir }}/secrets/icinga_api_pass",     length: 8 }
+    - { path: "{{ basedir }}/secrets/mysql_admin_pass",    length: 12 }
+    - { path: "{{ basedir }}/secrets/mysql_user_pass",     length: 12 }
+
+
+- name: Setup OAuth Client Info 
+  ansible.builtin.debug:
+    msg: "Client ID: {{ servicename }} // Client Secret: {{ oauth_client_secret }} // Redirect-URI: https://{{ domain }}/_oauth" 
+
+
+- name: pakete installieren
+  apt:
+    update_cache: no
+    state: present
+    name: 
+      - logrotate
+
+
+- name: icinga LogRotate config erstellen 
+  template: 
+    src: logrotate 
+    dest: /etc/logrotate.d/icinga
+
+
+- name: "create folder struct for {{ servicename }}"
+  file: 
+    path: "{{ item }}"
+    state: "directory"
+  with_items:
+    - "{{ basedir }}"
+    - "{{ basedir }}/secrets/"
+    - "{{ basedir }}/data/"
+    - "{{ basedir }}/etc/"
+    - "{{ basedir }}/log/"
+    - "{{ basedir }}/db/"
+    - "{{ basedir }}/graphite-conf/"
+    - "{{ basedir }}/graphite-storage/"
+
+
+- name: Konfig-Dateien erstellen (base)
+  template:
+    src: "{{ item }}"
+    dest: "{{ basedir }}/{{ item }}"
+  with_items:
+    - Dockerfile
+    - docker-compose.yml
+    - check_rbl_helper.sh
+    - notify_by_pushover.sh
+    - etc/locale.gen
+    - etc/oauth_header.conf
+  notify: restart icinga docker
+  register: dockerconfig
+
+- stat:
+    path: "{{ basedir }}/etc/icingaweb2/CONFIGURED"
+  register: configured
+
+- name: "start {{ servicename }} docker (init)"
+  community.docker.docker_compose_v2:
+    project_src: "{{ basedir }}"
+    state: present
+  when: configured.stat.exists == False
+
+- name: "wait for {{ servicename }} docker (init)"
+  wait_for:
+    path: "{{ basedir }}/etc/icingaweb2/CONFIGURED"
+  when: configured.stat.exists == False
+  
+- name: "stop {{ servicename }} docker (init)"
+  community.docker.docker_compose_v2:
+    project_src: "{{ basedir }}"
+    state: absent
+  when: configured.stat.exists == False
+
+- name: Script Helper erstellen
+  template: 
+    src: "{{ item }}" 
+    dest: "{{ basedir }}/{{ item }}"
+    mode: u+x
+  with_items:
+    - debuglog_enable.sh
+    - debuglog_disable.sh
+
+
+- name: Konfig-Dateien erstellen (icinga,icingaweb2,graphite)
+  template:
+    src: "{{ item }}"
+    dest: "{{ basedir }}/{{ item }}"
+  with_items:
+    - etc/icinga/conf.d/api-users.conf
+    - etc/icinga/conf.d/commands2.conf
+    - etc/icinga/conf.d/groups.conf
+    - etc/icinga/conf.d/hosts_manual.conf
+    - etc/icinga/conf.d/hosts.conf
+#    - etc/icinga/conf.d/notifications_pushover.conf
+    - etc/icinga/conf.d/notifications.conf
+    - etc/icinga/conf.d/services_backup.conf
+    - etc/icinga/conf.d/services_container.conf
+    - etc/icinga/conf.d/services_domains.conf
+    - etc/icinga/conf.d/services_exporters.conf
+    - etc/icinga/conf.d/services_mail.conf
+    - etc/icinga/conf.d/services_manual.conf
+#    - etc/icinga/conf.d/services_mqttsensors.conf
+    - etc/icinga/conf.d/services_system.conf
+    - etc/icinga/conf.d/services.conf
+    - etc/icinga/conf.d/templates.conf
+    - etc/icinga/conf.d/users_groups.conf
+    - etc/icinga/conf.d/users_sample.conf
+    - etc/icingaweb2/authentication.ini
+    - etc/icingaweb2/groups.ini
+    - etc/icingaweb2/resources.ini
+    - etc/icingaweb2/roles.ini
+  notify: restart icinga docker
+
+  
+- name: "start {{ servicename }} docker"
+  community.docker.docker_compose_v2:
+    project_src: "{{ basedir }}"
+    state: present
+    build: "{{ dockerconfig.changed }}"

testserver/docker_icinga/templates/Dockerfile

+FROM jordan/icinga2:2.14.0
+
+# Install additional Packages 
+RUN apt-get update \
+ && apt-get install -y -q --no-install-recommends \
+    curl \ 
+    dnsutils \
+    git \
+    jq \
+    libdata-validate-domain-perl \
+    libdata-validate-ip-perl \
+    libmonitoring-plugin-perl \
+    libnet-dns-perl \
+    libnet-ip-perl \
+    perl \
+    python3-requests \
+    python3 \
+    python3-paho-mqtt \
+ && apt-get autoremove -y \
+ && apt-get clean \
+ && rm -rf /tmp/* /var/lib/apt/lists/* /var/cache/debconf/*-old
+
+# Helper Scripe  
+COPY check_rbl_helper.sh /opt 
+COPY notify_by_pushover.sh /opt 
+RUN chmod +x /opt/*.sh
+
+# check_mqtt
+RUN cd /opt/ \
+ && git clone https://github.com/jpmens/check-mqtt.git \
+ && cd /opt/check-mqtt/ \
+ && git checkout v3.0
+
+# check_rbl
+RUN cd /opt/ \
+ && git clone https://github.com/matteocorti/check_rbl.git \
+ && cd /opt/check_rbl/ \
+ && git checkout v1.5.7
+
+# prom2json -- needed for check_metric_value
+RUN cd /opt/ \
+ && wget https://github.com/prometheus/prom2json/releases/download/v1.3.0/prom2json-1.3.0.linux-amd64.tar.gz \
+ && tar --strip-components=1 -xzvf prom2json-1.3.0.linux-amd64.tar.gz \
+ && chmod ugo+x /opt/prom2json
+ 
+# check_metric_value - commit from 16.12.2020
+RUN cd /opt/ \
+ && git clone https://github.com/elberfeld/check_metric_value.git \
+ && cd /opt/check_metric_value/ \
+ && git checkout b94d3c3e78497a05e3b4520d33421f37e4d77985
\ No newline at end of file

testserver/docker_icinga/templates/check_rbl_helper.sh

+#!/bin/bash
+/usr/bin/perl /opt/check_rbl/check_rbl --extra-opts=rbl@/opt/check_rbl/check_rbl.ini $@

testserver/docker_icinga/templates/debuglog_disable.sh

+#!/bin/sh 
+
+cd /srv/icinga 
+docker-compose exec app icinga2 feature disable debuglog
+docker-compose restart 
+rm log/icinga2/debug.log

testserver/docker_icinga/templates/debuglog_enable.sh

+#!/bin/sh 
+
+cd /srv/icinga 
+docker-compose exec app icinga2 feature enable debuglog
+docker-compose restart 
+tail -f log/icinga2/debug.log

testserver/docker_icinga/templates/docker-compose.yml

+
+version: "2.4"
+
+services:
+  
+  app:
+
+    build: .
+    restart: always
+    ports:
+      - "{{ api_port }}:5665"
+    volumes:
+      - "{{ basedir }}/data:/var/lib/icinga2"
+      - "{{ basedir }}/etc/locale.gen:/etc/locale.gen"
+      - "{{ basedir }}/etc/oauth_header.conf:/etc/apache2/conf-enabled/oauth_header.conf"
+      - "{{ basedir }}/etc/icinga:/etc/icinga2"
+      - "{{ basedir }}/etc/icingaweb2:/etc/icingaweb2"
+      - "{{ basedir }}/log/apache2:/var/log/apache2"
+      - "{{ basedir }}/log/icinga2:/var/log/icinga2"
+      - "{{ basedir }}/log/icingaweb2:/var/log/icingaweb2"
+    depends_on:
+      - db
+      - graphite
+    environment:
+      APACHE2_HTTP: BOTH
+      MYSQL_ROOT_PASSWORD: "{{ mysql_admin_pass }}"
+      MYSQL_PASSWORD: "{{ mysql_user_pass }}"
+      MYSQL_DATABASE: icinga
+      MYSQL_USER: icinga
+      DEFAULT_MYSQL_HOST: db
+      DEFAULT_MYSQL_USER: icinga
+      DEFAULT_MYSQL_PASS: "{{ mysql_user_pass }}"
+      ICINGAWEB2_ADMIN_PASS: "{{ icinga_admin_pass }}"
+      ICINGA2_FEATURE_GRAPHITE: 1
+      ICINGA2_FEATURE_GRAPHITE_HOST: graphite
+      ICINGA2_FEATURE_GRAPHITE_PORT: 2003
+      ICINGA2_FEATURE_DIRECTOR: 0
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.{{ servicename }}.middlewares={{ servicename }}-auth
+      - traefik.http.routers.{{ servicename }}.rule=Host(`{{ domain }}`)
+      - traefik.http.routers.{{ servicename }}.entrypoints=websecure
+      - traefik.http.services.{{ servicename }}.loadbalancer.server.port=80      
+    networks:
+      - default
+      - web
+
+
+  auth:
+    image: thomseddon/traefik-forward-auth:2.2
+    restart: always
+    environment: 
+      LOG_LEVEL: info
+      DEFAULT_ACTION: auth
+      DEFAULT_PROVIDER: generic-oauth
+      SECRET: {{ forward_auth_secret }}
+      PROVIDERS_GENERIC_OAUTH_AUTH_URL: {{ oauth_global.authorize_url }}
+      PROVIDERS_GENERIC_OAUTH_TOKEN_URL: {{ oauth_global.token_url }}
+      PROVIDERS_GENERIC_OAUTH_USER_URL: {{ oauth_global.userinfo_url }}
+      PROVIDERS_GENERIC_OAUTH_CLIENT_ID: {{ servicename }}
+      PROVIDERS_GENERIC_OAUTH_CLIENT_SECRET: {{ oauth_client_secret }}
+      PROVIDERS_GENERIC_OAUTH_SCOPE: profile
+      PROVIDERS_GENERIC_OAUTH_TOKEN_STYLE: header
+    labels:
+      - traefik.enable=true
+      - traefik.http.middlewares.{{ servicename }}-auth.forwardauth.address=http://auth:4181
+      - traefik.http.middlewares.{{ servicename }}-auth.forwardauth.authResponseHeaders=X-Forwarded-User
+      - traefik.http.services.{{ servicename }}-auth.loadbalancer.server.port=4181
+    networks:
+      - default
+      - web
+
+
+  db:
+
+    image: mariadb:10.7.1
+    restart: always
+    ports:
+      - "{{ int_ip4 }}:{{mysql_port}}:3306"
+    volumes:
+      - "{{ basedir }}/db:/var/lib/mysql"
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ mysql_admin_pass }}"
+      MYSQL_PASSWORD: "{{ mysql_user_pass }}"
+      MYSQL_DATABASE: icinga
+      MYSQL_USER: icinga
+    networks:
+      - default
+
+
+  graphite:
+    
+    image: graphiteapp/graphite-statsd:1.1.8-7
+    restart: always
+    volumes:
+      - "{{ basedir }}/graphite-conf:/opt/graphite/conf"
+      - "{{ basedir }}/graphite-storage:/opt/graphite/storage"
+    environment:
+      GRAPHITE_TIME_ZONE: "Europe/Berlin"
+      GRAPHITE_DATE_FORMAT: "%d.%m.%y"
+      GRAPHITE_LOG_FILE_INFO: "-"
+      GRAPHITE_LOG_FILE_EXCEPTION: "-"
+      GRAPHITE_LOG_FILE_CACHE: "-"
+      GRAPHITE_LOG_FILE_RENDERING: "-"
+    networks:
+      - default
+
+
+networks:
+  web:
+    external: true    
+  default:
+    driver: bridge
+    enable_ipv6: true
+    ipam:
+      driver: default
+      config:
+        # must be a ULA range
+        - subnet: fd00:dead:beef:{{ api_port }}::/64
\ No newline at end of file

testserver/docker_icinga/templates/etc/icinga/conf.d/api-users.conf

+/**
+ * The ApiUser objects are used for authentication against the API.
+ */
+object ApiUser "{{icinga_api_user}}" {
+  password = "{{icinga_api_pass}}"
+  permissions = [ "*" ]
+}

testserver/docker_icinga/templates/etc/icinga/conf.d/commands2.conf

+
+/**
+ * Check MQTT values
+ */
+
+object CheckCommand "check_mqtt" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check-mqtt/check-mqtt.py" ] 
+
+  arguments = {
+    "-H" = "$mqtt_host$"
+    "-u" = "$mqtt_user$"
+    "-p" = "$mqtt_password$"
+    "-P" = "$mqtt_port$"
+    "-a" = "$mqtt_cafile$"
+    "-C" = "$mqtt_certfile$"
+    "-k" = "$mqtt_keyfile$"
+    "-t" = "$mqtt_topic$"
+    "-m" = {
+      set_if = "$mqtt_max$"
+      value = "$mqtt_max$"
+    }
+
+    "-l" = "$mqtt_payload$"
+    "-v" = "$mqtt_value$"
+    "-o" = "$mqtt_operator$"
+
+    "-r" = {
+      set_if = "$mqtt_readonly$"
+      description = "Don't write."
+    }
+    "-n" = {
+      set_if = "$mqtt_insecure$"
+      description = "suppress TLS hostname check"
+    }
+  }
+}
+
+/**
+ * Check for Mail Blacklisting
+ */
+
+object CheckCommand "check_mail_blacklist" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check_rbl_helper.sh" ] 
+
+  arguments = {
+    "-H" = "$rbl_host$"
+    "-c" = "$rbl_critical$"
+    "-w" = "$rbl_warning$"
+  }
+}
+
+/**
+ * Check for Prometheus values 
+ */
+
+object CheckCommand "check_metric_value" {
+  import "plugin-check-command"
+
+  command = [ "/opt/check_metric_value/check_metric_value.py" ] 
+
+  arguments = {
+    "-P" = "/opt/prom2json"
+    "-U" = "$metric_url$"
+    "-M" = "$metric_name$"
+    "-n" = "$metric_labelname$"
+    "-v" = "$metric_labelvalue$"
+    "-o" = "$metric_operator$"
+    "-u" = "$metric_unit$"
+    "-w" = "$metric_warn$"
+    "-c" = "$metric_crit$"
+  }
+}
+
+/**
+ * Matrix Notification 
+ */
+
+object NotificationCommand "matrix-host-notification" {
+  import "plugin-notification-command"
+
+  command = [ "/opt/icinga2-matrix-bot/icinga2/matrix-host-notification.sh" ]
+
+  env = {
+    "NOTIFICATIONTYPE" = "$notification.type$"
+    "HOSTALIAS" = "$host.display_name$",
+    "HOSTADDRESS" = "$address$",
+    "HOSTNAME" = "$host.name$",
+    "HOSTSTATE" = "$host.state$",
+    "LONGDATETIME" = "$icinga.long_date_time$",
+    "HOSTOUTPUT" = "$host.output$",
+    "NOTIFICATIONAUTHORNAME" = "$notification.author$",
+    "NOTIFICATIONCOMMENT" = "$notification.comment$",
+    "HOSTDISPLAYNAME" = "$host.display_name$",
+    "ICINGA_WEBURL" = "$notification_icingaweb2url$",
+    "MATRIX_SERVER" = "$user.vars.matrix.server$",
+    "MATRIX_TOKEN" = "$user.vars.matrix.token$",
+    "MATRIX_CHANNEL" = "$user.vars.matrix.channel$",
+  }
+}
+
+object NotificationCommand "matrix-service-notification" {
+  import "plugin-notification-command"
+
+  command = [ "/opt/icinga2-matrix-bot/icinga2/matrix-service-notification.sh" ]
+
+  env = {
+    "NOTIFICATIONTYPE" = "$notification.type$"
+    "SERVICEDESC" = "$service.name$"
+    "HOSTALIAS" = "$host.display_name$",
+    "HOSTNAME" = "$host.name$",
+    "HOSTADDRESS" = "$address$",
+    "SERVICESTATE" = "$service.state$",
+    "LONGDATETIME" = "$icinga.long_date_time$",
+    "SERVICEOUTPUT" = "$service.output$",
+    "NOTIFICATIONAUTHORNAME" = "$notification.author$",
+    "NOTIFICATIONCOMMENT" = "$notification.comment$",
+    "HOSTDISPLAYNAME" = "$host.display_name$",
+    "SERVICEDISPLAYNAME" = "$service.display_name$",
+    "ICINGA_WEBURL" = "$notification_icingaweb2url$",
+    "MATRIX_SERVER" = "$user.vars.matrix.server$",
+    "MATRIX_TOKEN" = "$user.vars.matrix.token$",
+    "MATRIX_CHANNEL" = "$user.vars.matrix.channel$",
+  }
+}

testserver/docker_icinga/templates/etc/icinga/conf.d/groups.conf

+/**
+ * Host groups 
+ */
+
+object HostGroup "linux-servers" {
+  display_name = "Linux Servers"
+
+  assign where host.vars.os == "Linux"
+}
+
+/*
+object HostGroup "windows-servers" {
+  display_name = "Windows Servers"
+
+  assign where host.vars.os == "Windows"
+}
+*/
+
+object HostGroup "network" {
+  display_name = "Network Devices"
+}
+
+object HostGroup "other" {
+  display_name = "Other Devices"
+}
+
+/**
+ * Service groups by check command
+ */
+
+object ServiceGroup "ping" {
+  display_name = "Ping Checks"
+
+  assign where match("ping*", service.name)
+}
+
+object ServiceGroup "http" {
+  display_name = "HTTP Checks"
+
+  assign where match("http*", service.check_command)
+}
+
+object ServiceGroup "dns" {
+  display_name = "DNS Checks"
+
+  assign where match("dig*", service.check_command)
+}
+
+object ServiceGroup "mqtt" {
+  display_name = "MQTT Checks"
+
+  assign where match("check_mqtt*", service.check_command)
+}
+
+/**
+ * Service Goups assigned in Services
+ */
+
+object ServiceGroup "backup" {
+  display_name = "Backup Checks"
+}
+
+object ServiceGroup "container" {
+  display_name = "Docker Container Checks"
+}
+
+object ServiceGroup "ldap" {
+  display_name = "LDAP Checks"
+}
+
+object ServiceGroup "certificate" {
+  display_name = "Certificate Checks"
+}
+
+object ServiceGroup "mail" {
+  display_name = "Mail Checks"
+}
+
+object ServiceGroup "exporter" {
+  display_name = "Metrics Exporter Checks"
+}

testserver/docker_icinga/templates/etc/icinga/conf.d/hosts.conf

+
+{% for host in groups['{{ groupname }}'] %}
+{% if hostvars[host].alert is defined and hostvars[host].alert %}
+
+
+object Host "{{ host }}" {
+  import "generic-host"
+
+  address = "{{ hostvars[host].int_ip4 }}"
+
+  {% if hostvars[host].ext_ip4 is defined %}
+  vars.ext_ip4 = "{{ hostvars[host].ext_ip4 }}"
+  {% endif %}
+
+  {% if hostvars[host].ext_ip6 is defined %}
+  vars.ext_ip6 = "{{ hostvars[host].ext_ip6 }}"
+  {% endif %}
+
+  {% if host in monitoring.internal_ldap_servers %}
+  vars.is_ldapserver = "True"
+  {% endif %}
+
+  vars.os = "Linux"
+  vars.prometheus = true
+}
+
+{% endif %}
+{% endfor %}
+
+{% for host in monitoring.external_dns_servers %}
+
+object Host "ext_dns_{{ host.name }}" {
+  import "generic-host"
+
+  address = "{{ host.ip }}"
+  vars.is_dnsserver = "True"
+}
+
+{% endfor %}

testserver/docker_icinga/templates/etc/icinga/conf.d/hosts_manual.conf

+
+object Host "wz-router" {
+  import "generic-host"
+
+  address = "192.168.0.1"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-sw-core" {
+  import "generic-host"
+
+  address = "192.168.0.2"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-sw-access" {
+  import "generic-host"
+
+  address = "192.168.0.3"
+  
+  # Keine Notifications, da der Switch aus ist wenn die Zone geschlossen ist 
+  vars.enable_nofitications = false
+
+  groups = [ "network" ]
+}
+
+object Host "wz-sw-dach" {
+  import "generic-host"
+
+  address = "192.168.0.4"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-ap-dach" {
+  import "generic-host"
+
+  address = "192.168.0.13"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-ap-eingang" {
+  import "generic-host"
+
+  address = "192.168.0.14"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-ap-lounge" {
+  import "generic-host"
+
+  address = "192.168.0.10"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-ap-vortrag" {
+  import "generic-host"
+
+  address = "192.168.0.12"
+
+  groups = [ "network" ]
+}
+
+object Host "wz-ap-werkstatt" {
+  import "generic-host"
+
+  address = "192.168.0.11"
+
+  groups = [ "network" ]
+}
+
+object Host "eq3max-cube" {
+  import "generic-host"
+
+  address = "{{ eq3max.cube_host }}"
+
+  groups = [ "network" ]
+}

testserver/docker_icinga/templates/etc/icinga/conf.d/notifications.conf

+
+apply Notification "matrix" to Host {
+  import "matrix-host-notification"
+
+  users = [ "matrix" ]
+
+  interval = 4h
+
+  assign where host.address && host.vars.enable_nofitications != false
+}
+
+
+apply Notification "matrix" to Service {
+  import "matrix-service-notification"
+
+  users = [ "matrix" ]
+
+  interval = 4h
+
+  assign where service.name && service.vars.enable_nofitications != false
+}
+

testserver/docker_icinga/templates/etc/icinga/conf.d/notifications_pushover.conf

+
+object NotificationCommand "pushover-host-notification" { 
+  import "plugin-notification-command" 
+
+  command = [ "/opt/notify_by_pushover.sh" ] 
+
+  env = { 
+      PUSHOVERUSER = "$user.vars.pushover_user$" 
+      PUSHOVERTOKEN = "$user.vars.pushover_token$" 
+      PUSHOVERTITLE = "Icinga @ {{ inventory_hostname }}" 
+      PUSHOVERMESSAGE = "$notification.type$ $host.display_name$ $host.state$ $icinga.long_date_time$" 
+    } 
+} 
+
+object NotificationCommand "pushover-service-notification" { 
+  import "plugin-notification-command" 
+
+  command = [ "/opt/notify_by_pushover.sh" ] 
+
+  env = { 
+    PUSHOVERUSER = "$user.vars.pushover_user$" 
+    PUSHOVERTOKEN = "$user.vars.pushover_token$" 
+    PUSHOVERTITLE = "Icinga @ {{ inventory_hostname }}" 
+    PUSHOVERMESSAGE = "$notification.type$ $host.display_name$ $service.display_name$ $service.state$ $icinga.long_date_time$" 
+  } 
+} 
+
+template Notification "pushover-host-notification" { 
+  command = "pushover-host-notification" 
+
+  states = [ Up, Down ] 
+  types = [ Problem, FlappingStart ] 
+
+  period = "24x7" 
+} 
+
+template Notification "pushover-service-notification" { 
+  command = "pushover-service-notification" 
+
+  states = [ OK, Critical, Unknown ] 
+  types = [ Problem, FlappingStart ] 
+
+  period = "24x7" 
+} 
+
+apply Notification "pushover-icingaadmins" to Host { 
+  import "pushover-host-notification" 
+
+  user_groups = ["icingaadmins"] 
+
+  interval = 4h
+  times.begin = 15m
+
+  assign where host.address 
+} 
+
+apply Notification "pushover-icingaadmins" to Service { 
+  import "pushover-service-notification" 
+
+  user_groups = ["icingaadmins"] 
+
+  interval = 4h
+  times.begin = 15m
+
+  assign where service.name
+} 
+

testserver/docker_icinga/templates/etc/icinga/conf.d/services.conf

+
+apply Service "ping" {
+  import "generic-service"
+
+  check_command = "ping4"
+
+  vars.ping_wrta = "200"
+  vars.ping_crta = "500"
+  vars.ping_wpl = "60"
+  vars.ping_cpl = "90"
+
+  assign where host.address
+}
+
+apply Service "ping_ext_ip4" {
+  import "generic-service"
+
+  check_command = "ping4"
+
+  vars.ping_address = "$host.vars.ext_ip4$"
+
+  vars.ping_wrta = "100"
+  vars.ping_crta = "300"
+  vars.ping_wpl = "20"
+  vars.ping_cpl = "50"
+
+  assign where host.vars.ext_ip4
+}
+
+#apply Service "ping_ext_ip6" {
+#  import "generic-service"
+#
+#  check_command = "ping6"
+#
+#  vars.ping_address = "$host.vars.ext_ip6$"
+#
+#  vars.ping_wrta = "100"
+#  vars.ping_crta = "300"
+#  vars.ping_wpl = "20"
+#  vars.ping_cpl = "50"
+#
+#  assign where host.vars.ext_ip6
+#}
+
+apply Service "ssh" {
+  import "generic-service"
+
+  check_command = "ssh"
+
+  assign where host.address && host.vars.os == "Linux"
+}
+
+

testserver/docker_icinga/templates/etc/icinga/conf.d/services_backup.conf

+
+{% for host in groups['{{ groupname }}'] %}
+
+  {% if hostvars[host].borgbackup_repos is defined %}
+  {% for repo in hostvars[host].borgbackup_repos %}
+  {% if hostvars[host].borgbackup_repos[repo].alert is defined and hostvars[host].borgbackup_repos[repo].alert %}
+
+apply Service "borgbackup_age - {{repo}}" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9100/metrics"
+  vars.metric_name = "borgbackup_lastbackup"
+  vars.metric_labelname = "repo"
+  vars.metric_labelvalue = "{{repo}}"
+  vars.metric_operator = "lt-date"
+  vars.metric_unit = "hours"
+  vars.metric_warn = "{{hostvars[host].borgbackup_repos[repo].warning_age}}" 
+  vars.metric_crit = "{{hostvars[host].borgbackup_repos[repo].critical_age}}" 
+
+  groups = [ "backup" ]
+
+  assign where host.name == "{{host}}"
+}
+
+apply Service "borgbackup_count - {{repo}}" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9100/metrics"
+  vars.metric_name = "borgbackup_count"
+  vars.metric_labelname = "repo"
+  vars.metric_labelvalue = "{{repo}}"
+  vars.metric_operator = "lt"
+  vars.metric_warn = "{{hostvars[host].borgbackup_repos[repo].warning_count}}" 
+  vars.metric_crit = "{{hostvars[host].borgbackup_repos[repo].critical_count}}" 
+
+  groups = [ "backup" ]
+
+  assign where host.name == "{{host}}"
+}
+
+  {% endif %}
+  {% endfor %}
+  {% endif %}
+
+{% endfor %}

testserver/docker_icinga/templates/etc/icinga/conf.d/services_container.conf

+
+{% for host in groups['{{ groupname }}'] %}
+{% if hostvars[host].alert is defined and hostvars[host].alert %}
+
+  {% if hostvars[host].alert.containers is defined %}
+
+apply Service "docker_metrics" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = true
+
+  vars.http_address = "{{hostvars[host].int_ip4}}"
+  vars.http_port = 9323
+  vars.http_uri = "/metrics"
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true 
+}
+
+apply Service "docker_container_count_low" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9323/metrics"
+  vars.metric_name = "engine_daemon_container_states_containers"
+  vars.metric_labelname = "state"
+  vars.metric_labelvalue = "running"
+  vars.metric_operator = "lt"
+  vars.metric_warn = "{{ hostvars[host].alert.containers|length }}"
+  vars.metric_crit = "{{ hostvars[host].alert.containers|length }}"
+
+  groups = [ "container" ]
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true && host.vars.os == "Linux"
+}
+
+apply Service "docker_container_count_high" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9323/metrics"
+  vars.metric_name = "engine_daemon_container_states_containers"
+  vars.metric_labelname = "state"
+  vars.metric_labelvalue = "running"
+  vars.metric_operator = "gt"
+  vars.metric_warn = "{{ hostvars[host].alert.containers|length }}"
+  vars.metric_crit = "9999"
+
+  groups = [ "container" ]
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true && host.vars.os == "Linux"
+}
+
+apply Service "dockerstats_metrics" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = true
+
+  vars.http_address = "{{hostvars[host].int_ip4}}"
+  vars.http_port = 9487
+  vars.http_uri = "/metrics"
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true 
+}
+
+
+    {% for  container in hostvars[host].alert.containers %}
+
+
+apply Service "{{ container.name }} CPU" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9487/metrics"
+  vars.metric_name = "dockerstats_cpu_usage_ratio"
+  vars.metric_labelname = "name"
+  vars.metric_labelvalue = "{{container.name}}"
+  vars.metric_operator = "gt"
+  vars.metric_warn = "80" 
+  vars.metric_crit = "95" 
+
+  groups = [ "container" ]
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true && host.vars.os == "Linux"
+}
+
+apply Service "{{ container.name }} MEM" {
+  import "generic-service"
+
+  check_command = "check_metric_value"
+  enable_perfdata = true
+
+  vars.metric_url = "http://{{hostvars[host].int_ip4}}:9487/metrics"
+  vars.metric_name = "dockerstats_memory_usage_ratio"
+  vars.metric_labelname = "name"
+  vars.metric_labelvalue = "{{container.name}}"
+  vars.metric_operator = "gt"
+  vars.metric_warn = "80" 
+  vars.metric_crit = "95" 
+
+  groups = [ "container" ]
+
+  assign where host.name == "{{host}}" && host.vars.prometheus == true && host.vars.os == "Linux"
+}
+
+    {% endfor %}
+
+  {% endif %}
+
+{% endif %}
+{% endfor %}

testserver/docker_icinga/templates/etc/icinga/conf.d/services_domains.conf

+
+{% for host in groups['{{ groupname }}'] %}
+
+  {% if hostvars[host].ext_ip4 is defined and hostvars[host].ext_ip6 is defined and hostvars[host].webserver_domains is defined %}
+  {% for domain in hostvars[host].webserver_domains %}
+
+apply Service "http_ok - {{domain}}" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = false
+
+  vars.http_address = "{{domain}}"
+  vars.http_vhost = "{{domain}}"
+  vars.http_port = 80
+
+  assign where host.name == "{{host}}"
+}
+
+apply Service "http_301 - {{domain}}" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = false
+
+  vars.http_address = "{{domain}}"
+  vars.http_vhost = "{{domain}}"
+  vars.http_port = 80
+  vars.http_expect = "301 Moved Permanently"
+
+  assign where host.name == "{{host}}"
+}
+
+apply Service "https_ok - {{domain}}" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = false
+
+  vars.http_address = "{{domain}}"
+  vars.http_vhost = "{{domain}}"
+  vars.http_port = 443
+  vars.http_ssl = true
+  vars.http_sni = true
+
+  assign where host.name == "{{host}}"
+}
+
+apply Service "https_cert - {{domain}}" {
+  import "generic-service"
+
+  check_command = "http"
+  enable_perfdata = false
+
+  vars.http_address = "{{domain}}"
+  vars.http_vhost = "{{domain}}"
+  vars.http_port = 443
+  vars.http_ssl = true
+  vars.http_sni = true
+  vars.http_certificate = "10,5"
+
+  groups = [ "certificate" ]
+
+  assign where host.name == "{{host}}"
+}
+
+apply Service "dig(4) - {{domain}}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{domain}}"
+  vars.dig_record_type = "A"
+  vars.dig_expected_address = "{{ hostvars[host].ext_ip4 }}"
+  vars.dig_arguments = "+tcp"
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+apply Service "dig(6) - {{domain}}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{domain}}"
+  vars.dig_record_type = "AAAA"
+  vars.dig_expected_address = "{{ hostvars[host].ext_ip6 }}"
+  vars.dig_arguments = "+tcp"
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+  {% endfor %}
+  {% endif %}
+
+{% endfor %}
+
+
+{% for domain in global_domains %}
+
+apply Service "CAA record - {{ global_domains[domain].domain }}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{ global_domains[domain].domain }}"
+  vars.dig_record_type = "CAA"
+  vars.dig_expected_address = "letsencrypt.org"
+  vars.dig_arguments = "+tcp"
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+{% endfor %}
+
+
+{% for maildomain in mail_domains %}
+
+apply Service "mx record - {{ mail_domains[maildomain].maildomain }}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{ mail_domains[maildomain].maildomain }}"
+  vars.dig_record_type = "MX"
+  vars.dig_expected_address = "{{ mail_domains[maildomain].mxserver }}"
+  vars.dig_arguments = "+tcp"
+
+  groups = [ "mail" ]
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+apply Service "spf record - {{ mail_domains[maildomain].maildomain }}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{ mail_domains[maildomain].maildomain }}"
+  vars.dig_record_type = "TXT"
+  vars.dig_expected_address = "{{ mail_domains[maildomain].spf }}"
+  vars.dig_arguments = "+tcp"
+
+  groups = [ "mail" ]
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+  {% if mail_domains[maildomain].dmarc is defined %}
+
+apply Service "DMARC record - {{ mail_domains[maildomain].maildomain }}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "_dmarc.{{ mail_domains[maildomain].maildomain }}"
+  vars.dig_record_type = "TXT"
+  vars.dig_expected_address = "{{ mail_domains[maildomain].dmarc }}"
+  vars.dig_arguments = "+tcp"
+
+  groups = [ "mail" ]
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+  {% endif %}
+
+  {% if mail_domains[maildomain].dkim is defined %}
+  {% for entry in mail_domains[maildomain].dkim %}
+
+apply Service "DKIM {{entry.selector}} record - {{ mail_domains[maildomain].maildomain }}" {
+  import "generic-service"
+
+  check_command = "dig"
+  enable_perfdata = false
+
+  vars.dig_lookup = "{{entry.selector}}._domainkey.{{ mail_domains[maildomain].maildomain }}"
+  vars.dig_record_type = "TXT"
+  vars.dig_expected_address = "{{ entry.value | replace("\"","\\\"") }}"
+  vars.dig_arguments = "+tcp"
+
+  groups = [ "mail" ]
+
+  assign where host.address && host.vars.is_dnsserver == "True"
+}
+
+  {% endfor %}
+  {% endif %}
+
+{% endfor %}