diff --git a/testserver/docker_matrix/tasks/main.yml b/testserver/docker_matrix/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..27dc2fdaa156cef1d9699c849984ec3bff03650d --- /dev/null +++ b/testserver/docker_matrix/tasks/main.yml @@ -0,0 +1,65 @@ +--- + +- include_tasks: ../functions/get_secret.yml + with_items: + - { path: /srv/shared/noreply_email_pass, length: -1 } + - { path: /srv/ldap/secret/ldap_readonly_pass, length: -1 } + - { path: /srv/matrix/postgres_user_pass, length: 24 } + - { path: /srv/matrix/admin_access_token, length: -1 } # Get in Element fo an Admin User: Settings > Help > Advanced + + +- name: create folder struct for matrix + file: + path: "{{ item }}" + state: "directory" + owner: www-data + group: www-data + with_items: + - "/srv/matrix/" + - "/srv/matrix/ma1sd-config/" + - "/srv/matrix/ma1sd-data/" + - "/srv/matrix/synapse-data/" + + +- name: create folder struct for matrix db + file: + path: "{{ item }}" + state: "directory" + owner: "999" + group: "999" + with_items: + - "/srv/matrix/db/" + + +- name: Konfig-Dateien erstellen + template: + src: "{{ item }}" + dest: "/srv/matrix/{{ item }}" + with_items: + - docker-compose.yml + - rest_auth_provider.py + - ma1sd-config/ma1sd.yaml + - synapse-data/homeserver.log.config + - synapse-data/homeserver.yaml + register: configs + + +- name: Script-Dateien erstellen + template: + src: "{{ item }}" + dest: "/srv/matrix/{{ item }}" + mode: "ug+rwx" + with_items: + - purgemediacache.sh + + +- name: stop matrix docker + community.docker.docker_compose_v2: + project_src: /srv/matrix/ + state: absent + when: configs.changed + +- name: start matrix docker + community.docker.docker_compose_v2: + project_src: /srv/matrix/ + state: present diff --git a/testserver/docker_matrix/templates/README b/testserver/docker_matrix/templates/README new file mode 100644 index 0000000000000000000000000000000000000000..cbf322fa73a55334e3ec3f4c089fb82e5e50e400 --- /dev/null +++ b/testserver/docker_matrix/templates/README @@ -0,0 +1 @@ +the rest-auth_provider is from https://githubcom/ma1uta/matrix-synapse-rest-password-provider/ diff --git a/testserver/docker_matrix/templates/docker-compose.yml b/testserver/docker_matrix/templates/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..fb47a54e48dbd96f050c25dc540a6f6ef8db1f10 --- /dev/null +++ b/testserver/docker_matrix/templates/docker-compose.yml @@ -0,0 +1,90 @@ +services: + + # Datenbank Dump vor Upgrade (im Container) + # pg_dump --clean --create -d synapse -U synapse > /var/lib/postgresql/data/dump.sql + + # Datenbank einlesen nach Upgrade (im Container) + # psql -U synapse -d synapse < /var/lib/postgresql/data/dump.sql + + # Alternativer weg mit inplace-Upgrade (funktioniert nicht) + # https://github.com/tianon/docker-postgres-upgrade + + db: + + image: postgres:16 + restart: always + volumes: + - /srv/matrix/db:/var/lib/postgresql/data + environment: + POSTGRES_DB: synapse + POSTGRES_USER: synapse + POSTGRES_PASSWORD: "{{ postgres_user_pass }}" + POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C + networks: + - default + + synapse: + + image: matrixdotorg/synapse:latest + restart: always + cpu_count: "1" + cpuset: "0" + depends_on: + - db + - ma1sd + volumes: + - /srv/matrix/synapse-data/:/data + # Python version can be found in the dockerfile: https://github.com/matrix-org/synapse/blob/develop/docker/Dockerfile check for tag to get the correct version + - /srv/matrix/rest_auth_provider.py:/usr/local/lib/python3.11/site-packages/rest_auth_provider.py + environment: + SYNAPSE_CONFIG_PATH: "/data/homeserver.yaml" + TZ: "Europe/Berlin" + labels: + - traefik.enable=true + - traefik.http.routers.{{ servicename }}.rule=Host(`{{ domain }}`) + - traefik.http.routers.{{ servicename }}.entrypoints=websecure + - traefik.http.routers.{{ servicename }}.service={{ servicename }} + - traefik.http.services.{{ servicename }}.loadbalancer.server.port=8008 + - traefik.http.routers.matrix_federation.rule=Host(`{{ domain }}`) + - traefik.http.routers.matrix_federation.entrypoints=matrix_federation + - traefik.http.routers.matrix_federation.service=matrix_federation + - traefik.http.services.matrix_federation.loadbalancer.server.port=8448 + networks: + - default + - web + + ma1sd: + + image: ma1uta/ma1sd:2.5.0 + restart: always + volumes: + - /srv/matrix/ma1sd-config/:/etc/ma1sd + - /srv/matrix/ma1sd-data/:/var/ma1sd + labels: + - com.centurylinklabs.watchtower.enable=false + - traefik.enable=true + - traefik.http.routers.{{ servicename }}-ma1sd.rule=((Host(`{{ domain }}`) && PathPrefix(`/_matrix/client/r0/login`)) || (Host(`{{ domain }}`) && PathPrefix(`/_matrix/identity`))) + - traefik.http.routers.{{ servicename }}-ma1sd.entrypoints=websecure + - traefik.http.services.{{ servicename }}-ma1sd.loadbalancer.server.port=8090 + networks: + - default + - web + + + purgemediacache: + + image: jsonfry/curl-cron:latest + restart: always + depends_on: + - synapse + volumes: + - /srv/matrix/purgemediacache.sh:/curl.sh + environment: + CRON_SCHEDULE: "0 7 * * *" + networks: + - default + + +networks: + web: + external: true diff --git a/testserver/docker_matrix/templates/ma1sd-config/ma1sd.yaml b/testserver/docker_matrix/templates/ma1sd-config/ma1sd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd8b2422e75ea3deb49a769f8b038b0f9c44539b --- /dev/null +++ b/testserver/docker_matrix/templates/ma1sd-config/ma1sd.yaml @@ -0,0 +1,231 @@ + +####################### +# Matrix config items # +####################### +# Matrix domain, same as the domain configure in your Homeserver configuration. +# NOTE: in Synapse Homeserver, the Matrix domain is defined as 'server_name' in configuration file. +# +# This is used to build the various identifiers in all the features. +# +# If the hostname of the public URL used to reach your Matrix services is different from your Matrix domain, +# per example matrix.domain.tld vs domain.tld, then use the server.name configuration option. +# See the "Configure" section of the Getting Started guide for more info. +# +matrix: + domain: 'matrix.warpzone.ms' + v1: true # deprecated + v2: true # MSC2140 API v2. Riot require enabled V2 API. + + +################ +# Signing keys # +################ +# Absolute path for the Identity Server signing keys database. +# /!\ THIS MUST **NOT** BE YOUR HOMESERVER KEYS FILE /!\ +# If this path does not exist, it will be auto-generated. +# +# During testing, /var/tmp/ma1sd/keys is a possible value +# For production, recommended location shall be one of the following: +# - /var/lib/ma1sd/keys +# - /var/opt/ma1sd/keys +# - /var/local/ma1sd/keys +# +key: + path: '/var/ma1sd/keys' + + +# Path to the SQLite DB file for ma1sd internal storage +# /!\ THIS MUST **NOT** BE YOUR HOMESERVER DATABASE /!\ +# +# Examples: +# - /var/opt/ma1sd/store.db +# - /var/local/ma1sd/store.db +# - /var/lib/ma1sd/store.db +# +storage: +# backend: sqlite # or postgresql + provider: + sqlite: + database: '/var/ma1sd/store.db' +# postgresql: +# # Wrap all string values with quotes to avoid yaml parsing mistakes +# database: '//localhost/ma1sd' # or full variant //192.168.1.100:5432/ma1sd_database +# username: 'ma1sd_user' +# password: 'ma1sd_password' +# +# # Pool configuration for postgresql backend. +# ####### +# # Enable or disable pooling +# pool: false +# +# ####### +# # Check database connection before get from pool +# testBeforeGetFromPool: false # or true +# +# ####### +# # There is an internal thread which checks each of the database connections as a keep-alive mechanism. This set the +# # number of milliseconds it sleeps between checks -- default is 30000. To disable the checking thread, set this to +# # 0 before you start using the connection source. +# checkConnectionsEveryMillis: 30000 +# +# ####### +# # Set the number of connections that can be unused in the available list. +# maxConnectionsFree: 5 +# +# ####### +# # Set the number of milliseconds that a connection can stay open before being closed. Set to 9223372036854775807 to have +# # the connections never expire. +# maxConnectionAgeMillis: 3600000 + +################### +# Identity Stores # +################### +# If you are using synapse standalone and do not have an Identity store, +# see https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md#synapse-identity-store +# +# If you would like to integrate with your AD/Samba/LDAP server, +# see https://github.com/ma1uta/ma1sd/blob/master/docs/stores/ldap.md +# +# For any other Identity store, or to simply discover them, +# see https://github.com/ma1uta/ma1sd/blob/master/docs/stores/README.md + +ldap: + enabled: true + connection: + host: '{{ ldap_ip_ext }}' + port: 389 + bindDn: '{{ ldap_readonly_bind_dn }}' + bindPassword: '{{ ldap_readonly_pass }}' + baseDNs: + - '{{ ldap_base_dn }}' + filter: '(&(objectClass=inetOrgPerson)(memberof=CN=active,OU=groups,DC=warpzone,DC=ms))' + attribute: + uid: + type: 'uid' + value: 'uid' + name: 'uid' + threepid: + email: + - 'mail' + msisdn: + - 'phone' + +################################################# +# Notifications for invites/addition to profile # +################################################# +# This is mandatory to deal with anything e-mail related. +# +# For an introduction to sessions, invites and 3PIDs in general, +# see https://github.com/ma1uta/ma1sd/blob/master/docs/threepids/session/session.md#3pid-sessions +# +# If you would like to change the content of the notifications, +# see https://github.com/ma1uta/ma1sd/blob/master/docs/threepids/notification/template-generator.md +# +#### E-mail connector +threepid: + medium: + email: + identity: + # The e-mail to send as. + from: "matrix-identity@warpzone.ms" + + connectors: + smtp: + # SMTP host + host: "{{ smtp_host }}" + + # TLS mode for the connection + # Possible values: + # 0 Disable any kind of TLS entirely + # 1 Enable STARTLS if supported by server (default) + # 2 Force STARTLS and fail if not available + # 3 Use full TLS/SSL instead of STARTLS + # + tls: 1 + + # SMTP port + # Be sure to adapt depending on your TLS choice, if changed from default + port: "{{ smtp_port }}" + + # Login for SMTP + login: "{{ noreply_email_user }}" + + # Password for the account + password: "{{ noreply_email_pass }}" + + +#### MSC2134 (hash lookup) + +#hashing: +# enabled: false # enable or disable the hash lookup MSC2140 (default is false) +# pepperLength: 20 # length of the pepper value (default is 20) +# rotationPolicy: per_requests # or `per_seconds` how often the hashes will be updating +# hashStorageType: sql # or `in_memory` where the hashes will be stored +# algorithms: +# - none # the same as v1 bulk lookup +# - sha256 # hash the 3PID and pepper. +# delay: 2m # how often hashes will be updated if rotation policy = per_seconds (default is 10s) +# requests: 10 # how many lookup requests will be performed before updating hashes if rotation policy = per_requests (default is 10) + +### hash lookup for synapseSql provider. +# synapseSql: +# lookup: +# query: 'select user_id as mxid, medium, address from user_threepid_id_server' # query for retrive 3PIDs for hashes. +# legacyRoomNames: false # use the old query to get room names. + +### hash lookup for ldap provider (with example of the ldap configuration) +# ldap: +# enabled: true +# lookup: true # hash lookup +# activeDirectory: false +# defaultDomain: '' +# connection: +# host: 'ldap.domain.tld' +# port: 389 +# bindDn: 'cn=admin,dc=domain,dc=tld' +# bindPassword: 'Secret' +# baseDNs: +# - 'dc=domain,dc=tld' +# attribute: +# uid: +# type: 'uid' # or mxid +# value: 'cn' +# name: 'displayName' +# identity: +# filter: '(objectClass=inetOrgPerson)' + +#### MSC2140 (Terms) +#policy: +# policies: +# term_name: # term name +# version: 1.0 # version +# terms: +# en: # lang +# name: term name en # localized name +# url: https://ma1sd.host.tld/term_en.html # localized url +# fe: # lang +# name: term name fr # localized name +# url: https://ma1sd.host.tld/term_fr.html # localized url +# regexp: +# - '/_matrix/identity/v2/account.*' +# - '/_matrix/identity/v2/hash_details' +# - '/_matrix/identity/v2/lookup' +# + +# logging: +# root: error # default level for all loggers (apps and thirdparty libraries) +# app: info # log level only for the ma1sd +# requests: false # or true to dump full requests and responses + +dns: + overwrite: + homeserver: + client: + - name: 'matrix.warpzone.ms' + value: 'http://synapse:8008' + + +session: + policy: + validation: + enabled: false diff --git a/testserver/docker_matrix/templates/purgemediacache.sh b/testserver/docker_matrix/templates/purgemediacache.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd3f4f3e08117291b0acf9e756e7965cc9e70b32 --- /dev/null +++ b/testserver/docker_matrix/templates/purgemediacache.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +echo "$(date) - Start" + +TS_NOW=$(date +%s) +DELAY=$((30*24*60*60)) +TS=$((TS_NOW-$DELAY)) + +curl -X POST --insecure --header "Authorization: Bearer {{ admin_access_token }}" https://{{ domain }}/_synapse/admin/v1/purge_media_cache?before_ts=$(($TS*1000)) + +echo "$(date) End" \ No newline at end of file diff --git a/testserver/docker_matrix/templates/rest_auth_provider.py b/testserver/docker_matrix/templates/rest_auth_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..5f6f583a51c7691df6bdc6ae177a86778bac2640 --- /dev/null +++ b/testserver/docker_matrix/templates/rest_auth_provider.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# +# REST endpoint Authentication module for Matrix synapse +# Copyright (C) 2017 Kamax Sarl +# +# https://www.kamax.io/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import logging +from twisted.internet import defer +import requests +import json +import time + +logger = logging.getLogger(__name__) + + +class RestAuthProvider(object): + + def __init__(self, config, account_handler): + self.account_handler = account_handler + + if not config.endpoint: + raise RuntimeError('Missing endpoint config') + + self.endpoint = config.endpoint + self.regLower = config.regLower + self.config = config + + logger.info('Endpoint: %s', self.endpoint) + logger.info('Enforce lowercase username during registration: %s', self.regLower) + + @defer.inlineCallbacks + def check_password(self, user_id, password): + logger.info("Got password check for " + user_id) + data = {'user': {'id': user_id, 'password': password}} + r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json=data) + r.raise_for_status() + r = r.json() + if not r["auth"]: + reason = "Invalid JSON data returned from REST endpoint" + logger.warning(reason) + raise RuntimeError(reason) + + auth = r["auth"] + if not auth["success"]: + logger.info("User not authenticated") + defer.returnValue(False) + + localpart = user_id.split(":", 1)[0][1:] + logger.info("User %s authenticated", user_id) + + registration = False + if not (yield self.account_handler.check_user_exists(user_id)): + logger.info("User %s does not exist yet, creating...", user_id) + + if localpart != localpart.lower() and self.regLower: + logger.info('User %s was cannot be created due to username lowercase policy', localpart) + defer.returnValue(False) + + user_id, access_token = (yield self.account_handler.register(localpart=localpart)) + registration = True + logger.info("Registration based on REST data was successful for %s", user_id) + else: + logger.info("User %s already exists, registration skipped", user_id) + + if auth["profile"]: + logger.info("Handling profile data") + profile = auth["profile"] + + # fixme: temporary fix + try: + store = yield self.account_handler._hs.get_profile_handler().store # for synapse >= 1.9.0 + except AttributeError: + store = yield self.account_handler.hs.get_profile_handler().store # for synapse < 1.9.0 + + if "display_name" in profile and ((registration and self.config.setNameOnRegister) or (self.config.setNameOnLogin)): + display_name = profile["display_name"] + logger.info("Setting display name to '%s' based on profile data", display_name) + yield store.set_profile_displayname(localpart, display_name) + else: + logger.info("Display name was not set because it was not given or policy restricted it") + + if (self.config.updateThreepid): + if "three_pids" in profile: + logger.info("Handling 3PIDs") + + external_3pids = [] + for threepid in profile["three_pids"]: + medium = threepid["medium"].lower() + address = threepid["address"].lower() + external_3pids.append({"medium": medium, "address": address}) + logger.info("Looking for 3PID %s:%s in user profile", medium, address) + + validated_at = time_msec() + if not (yield store.get_user_id_by_threepid(medium, address)): + logger.info("3PID is not present, adding") + yield store.user_add_threepid( + user_id, + medium, + address, + validated_at, + validated_at + ) + else: + logger.info("3PID is present, skipping") + + if (self.config.replaceThreepid): + for threepid in (yield store.user_get_threepids(user_id)): + medium = threepid["medium"].lower() + address = threepid["address"].lower() + if {"medium": medium, "address": address} not in external_3pids: + logger.info("3PID is not present in external datastore, deleting") + yield store.user_delete_threepid( + user_id, + medium, + address + ) + + else: + logger.info("3PIDs were not updated due to policy") + else: + logger.info("No profile data") + + defer.returnValue(True) + + @staticmethod + def parse_config(config): + # verify config sanity + _require_keys(config, ["endpoint"]) + + class _RestConfig(object): + endpoint = '' + regLower = True + setNameOnRegister = True + setNameOnLogin = False + updateThreepid = True + replaceThreepid = False + + rest_config = _RestConfig() + rest_config.endpoint = config["endpoint"] + + try: + rest_config.regLower = config['policy']['registration']['username']['enforceLowercase'] + except TypeError: + # we don't care + pass + except KeyError: + # we don't care + pass + + try: + rest_config.setNameOnRegister = config['policy']['registration']['profile']['name'] + except TypeError: + # we don't care + pass + except KeyError: + # we don't care + pass + + try: + rest_config.setNameOnLogin = config['policy']['login']['profile']['name'] + except TypeError: + # we don't care + pass + except KeyError: + # we don't care + pass + + try: + rest_config.updateThreepid = config['policy']['all']['threepid']['update'] + except TypeError: + # we don't care + pass + except KeyError: + # we don't care + pass + + try: + rest_config.replaceThreepid = config['policy']['all']['threepid']['replace'] + except TypeError: + # we don't care + pass + except KeyError: + # we don't care + pass + + return rest_config + + +def _require_keys(config, required): + missing = [key for key in required if key not in config] + if missing: + raise Exception( + "REST Auth enabled but missing required config values: {}".format( + ", ".join(missing) + ) + ) + + +def time_msec(): + """Get the current timestamp in milliseconds + """ + return int(time.time() * 1000) diff --git a/testserver/docker_matrix/templates/synapse-data/homeserver.log.config b/testserver/docker_matrix/templates/synapse-data/homeserver.log.config new file mode 100644 index 0000000000000000000000000000000000000000..c2fef97c0c0a13db9575218a4ce51daaad02f470 --- /dev/null +++ b/testserver/docker_matrix/templates/synapse-data/homeserver.log.config @@ -0,0 +1,29 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: "WARNING" + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: "WARNING" + +root: + level: "WARNING" + handlers: [console] diff --git a/testserver/docker_matrix/templates/synapse-data/homeserver.yaml b/testserver/docker_matrix/templates/synapse-data/homeserver.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93813866e74f62ffc18298e85b73cf81e0581264 --- /dev/null +++ b/testserver/docker_matrix/templates/synapse-data/homeserver.yaml @@ -0,0 +1,121 @@ +server_name: "{{ matrix.domain }}" +pid_file: /tmp/homeserver.pid +public_baseurl: "{{ matrix.public_url }}/" +use_presence: false +allow_public_rooms_without_auth: false +allow_public_rooms_over_federation: true +forget_rooms_on_leave: true + +listeners: + - port: 8448 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + + - port: 8008 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false + +admin_contact: 'mailto:verwaltung@warpzone.ms' + +retention: + enabled: true + + +database: + name: "psycopg2" + args: + user: synapse + password: "{{ postgres_user_pass }}" + database: synapse + host: db + cp_min: 5 + cp_max: 10 + +log_config: "/data/homeserver.log.config" + +media_store_path: "/data/media_store" +max_upload_size: 10M +dynamic_thumbnails: false +thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale + +url_preview_enabled: true +url_preview_ip_range_blacklist: + - '127.0.0.0/8' + - '10.0.0.0/8' + - '172.16.0.0/12' + - '192.168.0.0/16' + - '100.64.0.0/10' + - '192.0.0.0/24' + - '169.254.0.0/16' + - '198.18.0.0/15' + - '192.0.2.0/24' + - '198.51.100.0/24' + - '203.0.113.0/24' + - '224.0.0.0/4' + - '::1/128' + - 'fe80::/10' + - 'fc00::/7' + +max_spider_size: 10M + +enable_registration: false +default_identity_server: "{{ matrix.identity_server }}" + +auto_join_rooms: + - "#warpzone:{{ matrix.domain }}" + +report_stats: false + +signing_key_path: "/data/homeserver.signing.key" +key_refresh_interval: 1d +suppress_key_server_warning: true +trusted_key_servers: + - server_name: "matrix.org" + +email: + smtp_host: {{ smtp_host }} + smtp_port: {{ smtp_port }} + smtp_user: "{{ noreply_email_user }}" + smtp_pass: "{{ noreply_email_pass }}" + require_transport_security: false + notif_from: "Warpzone Matrix <matrix@{{ smtp_domain }}>" + enable_notifs: true + notif_for_new_users: False + +password_providers: + - module: "rest_auth_provider.RestAuthProvider" + config: + endpoint: "http://ma1sd:8090" + +encryption_enabled_by_default_for_room_type: invite +enable_group_creation: false + +user_directory: + enabled: true + search_all_users: false + + + # new in 1.34 spaces + experimental_features: { spaces_enabled: true }