deployed nodered, grafana, prometheus on warpsrvint

common/prometheus-node/tasks/main.yml

+---
+# Pakete installieren
+- name: pakete installieren
+  apt:
+    pkg: "{{ item }}"
+    update_cache: yes
+    state: installed
+  with_items:
+    - prometheus-node-exporter
+
+- name: reload systemd and enable service
+  systemd:
+    name: prometheus-node-exporter
+    enabled: yes
+    daemon_reload: yes
+
+- name: restart prometheus-node-exporter
+  service: name=prometheus-node-exporter state=restarted

host_vars/warpsrvint

@@ -10,7 +10,7 @@ debian_sources:
   - "deb http://security.debian.org/ jessie/updates main contrib non-free"
   - "deb http://debian.uni-duisburg-essen.de/debian/ jessie-updates main contrib non-free"
   - "deb https://apt.dockerproject.org/repo debian-jessie main"
-  - "deb http://http.debian.net/debian wheezy-backports main"
+  - "deb http://http.debian.net/debian jessie-backports main"
 
 debian_keys:
 

warpsrvint/docker_grafana/tasks/main.yml

+---
+# Einige Secrets sind auf dem Server lokal gespeichert und werden von dort gelesen 
+# Auslesen der Dateien vom Server, zwischengespeicert wird in der Variable gitlab_secrets 
+# Anschließend müssen die entsprechenden Einträge aus gitlab_secrets extrahiert werden  
+# Die Daten, die von Slurp gelesen werden sind Base64 codiert 
+# Zur Sicherheit werden Whitespace-Zeichen entfert, damit z.B. Zeilenumbrüche nicht übernommen werden 
+
+- name: get secrets from server 1
+  slurp: src={{ item }}
+  with_items:
+    - /srv/ldap/secret/ldap_readonly_pass
+  register: warpinfra_secrets
+
+- name: get secrets from server 2
+  set_fact: 
+    ldap_readonly_pass: "{{ warpinfra_secrets.results | selectattr('item', 'equalto', '/srv/ldap/secret/ldap_readonly_pass') | map(attribute='content') | list | first | b64decode | regex_replace('\\s', '') }}" 
+
+- name: get secrets from server 1
+  slurp: src={{ item }}
+  with_items:
+    - /srv/grafana/grafana_admin_pass
+  register: grafana_secrets
+
+- name: get secrets from server 2
+  set_fact: 
+    grafana_admin_pass: "{{ grafana_secrets.results | selectattr('item', 'equalto', '/srv/grafana/grafana_admin_pass') | map(attribute='content') | list | first | b64decode | regex_replace('\\s', '') }}" 
+
+- name: create folder struct for grafana
+  file: 
+    path: "{{ item }}"
+    state: "directory"
+  with_items:
+    - "/srv/grafana/"
+    - "/srv/grafana/config/"
+    - "/srv/grafana/data/"
+
+- name: create config files
+  template: src={{ item }} dest=/srv/grafana/config/{{ item }}
+  with_items: 
+    - grafana.ini
+    - ldap.toml
+  register: config
+
+- name: stop grafana-app docker
+  docker_container: 
+    name: grafana-app
+    state: absent
+  when: config.changed
+
+- name: start grafana-app docker
+  docker_container: 
+    name: grafana-app
+    image: grafana/grafana:4.4.1
+    state: started
+    restart_policy: always
+    volumes:
+      - /srv/grafana/config/grafana.ini:/etc/grafana/grafana.ini
+      - /srv/grafana/config/ldap.toml:/etc/grafana/ldap.toml
+      - /srv/grafana/data/:/var/lib/grafana
+    ports:
+      - 3000:3000
+    env:
+      GF_SERVER_ROOT_URL: "http://10.5.0.111:3000"
+      GF_SECURITY_ADMIN_PASSWORD: "{{ grafana_admin_pass }}"
+      

warpsrvint/docker_grafana/templates/grafana.ini

+##################### Grafana Configuration ##################################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+instance_name = intern
+
+#
+#################################### Server ####################################
+[server]
+# Protocol (http, https, socket)
+;protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port  to use
+;http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+;root_url = http://localhost:3000
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket path
+;socket =
+
+
+#################################### Security ####################################
+[security]
+# default admin user, created on startup
+;admin_user = admin
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+;admin_password = admin
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# Auto-login remember days
+;login_remember_days = 7
+;cookie_username = grafana_user 
+;cookie_remember_name = grafana_remember
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+# remove snapshots after 90 days
+;snapshot_TTL_days = 90
+
+#################################### Users ####################################
+[users]
+# disable user signup / registration
+allow_sign_up = false
+
+# Allow non admin users to create organizations
+allow_org_create = false
+
+# Set to true to automatically assign new users to the default organization (id 1)
+auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+auto_assign_org_role = Viewer
+
+# Background text for the user field on the login page
+login_hint = infa.warpzone.ms account
+
+# Default UI theme ("dark" or "light")
+default_theme = dark
+
+[auth]
+# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
+;disable_login_form = false
+
+# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
+;disable_signout_menu = false
+
+#################################### Anonymous Auth ##########################
+[auth.anonymous]
+# enable anonymous access
+enabled = true
+
+# specify organization name that should be used for unauthenticated users
+org_name = Main Org.
+
+# specify role for unauthenticated users
+org_role = Viewer
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+enabled = true
+config_file = /etc/grafana/ldap.toml
+allow_sign_up = true
+
+
+#################################### Alerting ############################
+[alerting]
+# Disable alerting engine & UI features
+enabled = false
+# Makes it possible to turn off alert rule execution but alerting UI is visible
+execute_alerts = false
+

warpsrvint/docker_grafana/templates/ldap.toml

+# Set to true to log user information returned from LDAP
+verbose_logging = false
+
+[[servers]]
+# Ldap server host (specify multiple hosts space separated)
+host = "10.0.20.2"
+# Default port is 389 or 636 if use_ssl = true
+port = 389
+# Set to true if ldap server supports TLS
+use_ssl = false
+# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS)
+start_tls = false
+# set to true if you want to skip ssl cert validation
+ssl_skip_verify = false
+# set to the path to your root CA certificate or leave unset to use system defaults
+# root_ca_cert = "/path/to/certificate.crt"
+
+# Search user bind dn
+bind_dn = "cn=readonly,dc=warpzone,dc=ms"
+# Search user bind password
+# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
+bind_password = '{{ldap_readonly_pass}}'
+
+# User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"
+search_filter = "(uid=%s)"
+
+# An array of base dns to search through
+search_base_dns = ["dc=warpzone,dc=ms"]
+
+# In POSIX LDAP schemas, without memberOf attribute a secondary query must be made for groups.
+# This is done by enabling group_search_filter below. You must also set member_of= "cn"
+# in [servers.attributes] below.
+
+## Group search filter, to retrieve the groups of which the user is a member (only set if memberOf attribute is not available)
+# group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))"
+## An array of the base DNs to search through for groups. Typically uses ou=groups
+# group_search_base_dns = ["ou=groups,dc=grafana,dc=org"]
+
+# Specify names of the ldap attributes your ldap uses
+[servers.attributes]
+name = "givenName"
+surname = "sn"
+username = "uid"
+member_of = "memberOf"
+email =  "email"
+
+# Map ldap groups to grafana org roles
+[[servers.group_mappings]]
+group_dn = "cn=grafana-admin,ou=infrastructure,dc=warpzone,dc=ms"
+org_role = "Admin"
+
+[[servers.group_mappings]]
+group_dn = "cn=active,ou=groups,dc=warpzone,dc=ms"
+org_role = "Editor"
+
+[[servers.group_mappings]]
+# If you want to match all (or no ldap groups) then you can use wildcard
+group_dn = "*"
+org_role = "Viewer"

warpsrvint/docker_nodered/tasks/main.yml

+---
+- name: create folder struct for nodered
+  file: 
+    path: "/srv/nodered/data/" 
+    state: "directory"  
+     
+- name: start nodered docker
+  docker_container: 
+    name: nodered-app
+    image: nodered/node-red-docker:0.16.2
+    state: started
+    restart_policy: always
+    volumes:
+      - /srv/nodered/data/:/data
+    ports:
+      - 1880:1880
+
+    
\ No newline at end of file

warpsrvint/docker_prometheus/tasks/main.yml

+---
+
+- name: create folder struct for prometheus
+  file: 
+    path: "{{ item }}"
+    state: "directory"
+  with_items:
+    -  /srv/prometheus/
+    -  /srv/prometheus/config/
+    -  /srv/prometheus/data/
+    -  /srv/prometheus/alert-data/
+
+- name: create config files
+  template: src={{ item }} dest=/srv/prometheus/config/{{ item }}
+  with_items: 
+    - alertmanager.yml
+    - prometheus.yml
+    - prometheus.rules
+  register: config
+
+- name: start prometheus blackbox-exporter docker
+  docker_container: 
+    name: prometheus-blackbox-exporter
+    image: prom/blackbox-exporter:v0.5.0
+    state: started
+    restart_policy: always
+    ports:
+      - 0.0.0.0:9115:9115
+
+- name: start prometheus snmp-exporter docker
+  docker_container: 
+    name: prometheus-snmp-exporter
+    image: prom/snmp-exporter:v0.4.0
+    state: started
+    restart_policy: always
+    ports:
+      - 0.0.0.0:9116:9116
+
+- name: start prometheus statsd-exporter docker
+  docker_container: 
+    name: prometheus-statsd-exporter
+    image: prom/statsd-exporter:v0.4.0
+    state: started
+    restart_policy: always
+    ports:
+      - 0.0.0.0:9102:9102
+      - 0.0.0.0:9125:9125/udp
+
+- name: stop prometheus-alertmanager docker
+  docker_container: 
+    name: prometheus-alert
+    state: absent
+  when: config.changed
+
+- name: start prometheus-alertmanager docker
+  docker_container: 
+    name: prometheus-alert
+    image: prom/alertmanager:v0.7.1
+    state: started
+    restart_policy: always
+    volumes:
+      - /srv/prometheus/config/alertmanager.yml/:/etc/alertmanager/config.yml
+      - /srv/prometheus/alert-data/:/alertmanager
+    ports:
+      - 0.0.0.0:9093:9093
+
+- name: stop prometheus docker
+  docker_container: 
+    name: prometheus-app
+    state: absent
+  when: config.changed
+
+- name: start prometheus docker
+  docker_container: 
+    name: prometheus-app
+    image: prom/prometheus:v1.7.1
+    state: started
+    restart_policy: always
+    volumes:
+      - /srv/prometheus/config/prometheus.yml/:/etc/prometheus/prometheus.yml
+      - /srv/prometheus/config/prometheus.rules/:/etc/prometheus/prometheus.rules
+      - /srv/prometheus/data/:/prometheus
+    ports:
+      - 0.0.0.0:9090:9090
+    links:
+      - prometheus-blackbox-exporter:blackbox-exporter
+      - prometheus-snmp-exporter:snmp-exporter
+      - prometheus-statsd-exporter:statsd-exporter
+      - prometheus-alert:alertmanager
+

warpsrvint/docker_prometheus/templates/alertmanager.yml

+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'mail.warpzone.ms:25'
+  smtp_from: 'alert@warpzone.ms'
+  # smtp_auth_username: 'alertmanager'
+  # smtp_auth_password: 'password'
+
+# The root route on which each incoming alert enters.
+route:
+  # The labels by which incoming alerts are grouped together. For example,
+  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
+  # be batched into a single group.
+  group_by: ['alertname', 'cluster', 'service']
+
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first 
+  # notification.
+  group_wait: 30s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 5m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 3h 
+
+  # A default receiver
+  receiver: mail
+
+
+# Inhibition rules allow to mute a set of alerts given that another alert is
+# firing.
+# We use this to mute any warning-level notifications if the same alert is 
+# already critical.
+inhibit_rules:
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  # Apply inhibition if the alertname is the same.
+  equal: ['alertname', 'cluster', 'service']
+
+receivers:
+- name: 'mail'
+  email_configs:
+  - to: 'void@members.warpzone.ms'

warpsrvint/docker_prometheus/templates/prometheus.rules

+
+ALERT MEM_FULL_99P
+IF ((node_memory_MemTotal - node_memory_MemFree )/ node_memory_MemTotal * 100) > 99
+FOR 15m
+
+ALERT HIGH_CPU
+IF rate(node_cpu{mode = "idle"}[5m]) < 0.1 
+FOR 15m
+
+ALERT HIGH_DISK_IO
+IF node_disk_io_now > 10 
+FOR 5m
+
+ALERT DISK_FULL_95P 
+IF ( 100 *(1 - (node_filesystem_free  / node_filesystem_size) ) ) > 95 
+
+ALERT DISK_FULL_4H
+IF predict_linear(node_filesystem_free[1h],4*3600) < 0

warpsrvint/docker_prometheus/templates/prometheus.yml

+# my global config
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'prometheus'
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  - "/etc/prometheus/prometheus.rules"
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets:
+      - "alertmanager:9093"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    static_configs:
+      - targets: ['localhost:9090']
+        labels:
+          group: 'service'
+
+
+  - job_name: 'node'
+
+    static_configs:
+      - targets: ['warpsrvint:9100']
+        labels:
+          group: 'server'
+
+
+  - job_name: 'snmp'
+
+    metrics_path: /snmp
+    params:
+      module: [default]
+    static_configs:
+      - targets:
+        - warpfire  
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: snmp-exporter:9116  
+
+
+  - job_name: 'ping'
+    metrics_path: /probe
+    params:
+      module: [icmp]  
+    static_configs:
+      - targets:
+        - 212.124.34.241         # Next Hop Globe 
+        - 2001:470:1f0a:a3b::1   # HE Tunnel Endpoint 
+        - 8.8.8.8                # Google DNS 
+        - 217.79.181.126         # Server MyLoc IPv4 
+        - 2001:4ba0:ffff:7c::1   # Server MyLoc IPv4 
+        - 10.5.0.1               # warpfire
+        - 192.168.0.100          # Switch HP 
+        - 192.168.0.101          # Switch Brocade 
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115  
+
+
+  - job_name: 'http'
+    metrics_path: /probe
+    params:
+      module: [http_2xx]  
+    static_configs:
+      - targets:
+        - https://warpzone.ms  
+        - https://gitlab.warpzone.ms  
+        - https://infra.warpzone.ms  
+        - https://mattermost.warpzone.ms  
+        - https://pad.warpzone.ms  
+        - https://wiki.warpzone.ms  
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115  

warpsrvint/main.yml

@@ -3,10 +3,14 @@
 - hosts: warpsrvint
   remote_user: root
   roles:
+    - { role: "../common/prometheus-node", tags: prometheus-node }
     - { role: nginx, tags: nginx }
     - { role: docker, tags: docker }
+    - { role: docker_grafana, tags: grafana }
 #    - { role: docker_ldap, tags: ldap }
     - { role: docker_mqtt, tags: mqtt }
+    - { role: docker_nodered, tags: nodered }
+    - { role: docker_prometheus, tags: prometheus }
     - { role: docker_warpinfra, tags: warpinfra }
     - { role: docker_warpinfratest, tags: warpinfratest }