Skip to content
Snippets Groups Projects
Commit da60f611 authored by void's avatar void
Browse files

modul für verteilte borgbackups

parent 66165f86
No related branches found
No related tags found
No related merge requests found
Showing with 292 additions and 45 deletions
...@@ -9,40 +9,122 @@ ...@@ -9,40 +9,122 @@
- borgbackup - borgbackup
- logrotate - logrotate
- moreutils - moreutils
- openssl
- name: create directories 1
file:
path: "{{ item }}"
state: "directory"
with_items:
- /srv/borgbackup
- /var/log/borgbackup
- name: create directories 2
file:
path: "/srv/borgbackup/{{ item.key }}"
state: "directory"
with_dict: "{{ borgbackup_repos }}"
- name: generate new repo_passphrase (check)
stat:
path: /srv/borgbackup/repo_passphrase
register: repo_passphrase
- name: generate new repo_passphrase (generate)
command: openssl rand -base64 -out /srv/borgbackup/repo_passphrase 24
when: repo_passphrase.stat.exists == False
- name: generate new repo_sshkey (check)
stat:
path: /srv/borgbackup/repo_sshkey
register: repo_sshkey
- name: generate new repo_sshkey (generate)
command: ssh-keygen -N '' -t ed25519 -f /srv/borgbackup/repo_sshkey
when: repo_sshkey.stat.exists == False
- name: get secrets from server 1 - name: get secrets from server 1
slurp: src={{ item }} slurp: src={{ item }}
with_items: with_items:
- /srv/borgbackup/repo_passphrase - /srv/borgbackup/repo_passphrase
- /srv/borgbackup/repo_url
register: borgbackup_secrets register: borgbackup_secrets
- name: get secrets from server 2 - name: get secrets from server 2
set_fact: set_fact:
repo_passphrase: "{{ borgbackup_secrets.results | selectattr('item', 'equalto', '/srv/borgbackup/repo_passphrase') | map(attribute='content') | list | first | b64decode | regex_replace('\\s', '') }}" repo_passphrase: "{{ borgbackup_secrets.results | selectattr('item', 'equalto', '/srv/borgbackup/repo_passphrase') | map(attribute='content') | list | first | b64decode | regex_replace('\\s', '') }}"
repo_url: "{{ borgbackup_secrets.results | selectattr('item', 'equalto', '/srv/borgbackup/repo_url') | map(attribute='content') | list | first | b64decode | regex_replace('\\s', '') }}"
# BorgBackup Scripte erstellen # BorgBackup Scripte erstellen
- name: BorgBackup Script erstellen - name: BorgBackup Scripte erstellen (check)
template: src={{ item }} dest=/opt/{{ item }} mode=o+x template:
with_items: src: borgbackup-check.sh
- "borgbackup-check.sh" dest: /srv/borgbackup/{{ item.key }}/borgbackup-check.sh
- "borgbackup-create.sh" mode: o+x
- "borgbackup-delete.sh" with_dict: "{{ borgbackup_repos }}"
- "borgbackup-info.sh"
- "borgbackup-init.sh" - name: BorgBackup Scripte erstellen (create)
- "borgbackup-list.sh" template:
- "borgbackup-mount.sh" src: borgbackup-create.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-create.sh
- name: BorgBackup log folder erstellen mode: o+x
file: with_dict: "{{ borgbackup_repos }}"
path: "/var/log/borgbackup"
state: "directory" - name: BorgBackup Scripte erstellen (delete)
template:
src: borgbackup-delete.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-delete.sh
mode: o+x
with_dict: "{{ borgbackup_repos }}"
- name: BorgBackup Scripte erstellen (init)
template:
src: borgbackup-init.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-init.sh
mode: o+x
with_dict: "{{ borgbackup_repos }}"
- name: BorgBackup Scripte erstellen (info)
template:
src: borgbackup-info.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-info.sh
mode: o+x
with_dict: "{{ borgbackup_repos }}"
- name: BorgBackup Scripte erstellen (list)
template:
src: borgbackup-list.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-list.sh
mode: o+x
with_dict: "{{ borgbackup_repos }}"
- name: BorgBackup Scripte erstellen (mount)
template:
src: borgbackup-mount.sh
dest: /srv/borgbackup/{{ item.key }}/borgbackup-mount.sh
mode: o+x
with_dict: "{{ borgbackup_repos }}"
- name: BorgBackup Scripte erstellen (prometheus)
template:
src: borgbackup-prometheus.sh
dest: /srv/borgbackup/borgbackup-prometheus.sh
mode: o+x
- name: BorgBackup LogRotate config erstellen - name: BorgBackup LogRotate config erstellen
template: src=logrotate dest=/etc/logrotate.d/borgbackup template:
src: logrotate
dest: /etc/logrotate.d/borgbackup
- name: Alten cronjob entfernen falls vorhanden
cron:
name: "borgbackup"
state: absent
- name: Cronjob für BorgBackup Backup
cron: name="borgbackup-{{ item.key }}" weekday="{{ item.value.weekday }}" hour="{{ item.value.hour }}" minute="{{ item.value.minute }}" job="/srv/borgbackup/{{ item.key }}/borgbackup-create.sh 2>&1 | ts '[\\%Y-\\%m-\\%d \\%H:\\%M:\\%S]' >> /var/log/borgbackup/{{ item.key }}.log"
with_dict: "{{ borgbackup_repos }}"
- name: Cronjob für BorgBackup - name: Cronjob für BorgBackup Prometheus export
cron: name="borgbackup" weekday="{{borgbackup_weekday}}" hour="{{borgbackup_hour}}" minute="{{borgbackup_minute}}" job="/opt/borgbackup-create.sh 2>&1 | ts '[%Y-%m-%d %H:%M:%S]' >> /var/log/borgbackup/borgbackup.log" cron: name="borgbackup-prom" weekday="*" hour="*" minute="40" job="/srv/borgbackup/borgbackup-prometheus.sh 2>&1 | ts '[\\%Y-\\%m-\\%d \\%H:\%M:\\%S]' >> /var/log/borgbackup/borgbackup-prometheus.log"
\ No newline at end of file
#!/bin/bash #!/bin/bash
# Überprüfung der Backup Archive
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
echo "===[ Check Repo: {{ item.value.repo }} ]==="
borg check $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }}
# Überprüfung des Archives
borg check $1 $2 $3 --info --show-rc {{repo_url}}
#!/bin/bash #!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
export BACKUP_DATE=`date +%Y-%m-%d_%H_%M`
# Ausführung des Backups # Ausführung der Backups
# anschließend Bereinigung # anschließend Bereinigung
# abschließend Integritätscheck # abschließend Integritätscheck
borg create $1 $2 $3 --info --show-rc --stats --compression lzma,2 {{repo_url}}::`date +%Y-%m-%d_%H_%M` \ echo "===[ Create Backup: {{ item.value.repo }} ]===" \
&& \
borg create $1 $2 $3 --info --show-rc --stats --compression {{ item.value.compression }} {{ item.value.options }} {{ item.value.repo }}::$BACKUP_DATE \
{% for directory in borgbackup_directories %} {% for directory in borgbackup_directories %}
{{ directory }} \ {{ directory }} \
{% endfor %} {% endfor %} \
{% if item.value.directories is defined %}
{% for directory in item.value.directories %}
{{ directory }} \
{% endfor %} \
{% endif %}
&& \
echo "===[ Prune old Backups: {{ item.value.repo }} ]===" \
&& \
borg prune $1 $2 $3 --info --show-rc --list {{ item.value.prune }} {{ item.value.options }} {{ item.value.repo }} \
&& \
echo "===[ Check Repo: {{ item.value.repo }} ]===" \
&& \ && \
borg prune $1 $2 $3 --info --show-rc --list {{repo_url}} \ borg check $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }} \
{% for prune in borgbackup_prune %}
{{ prune }} \
{% endfor %}
&& \ && \
borg check $1 $2 $3 --info --show-rc {{repo_url}} date > "/srv/borgbackup/{{ item.key }}/lastbackup"
#!/bin/bash #!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
# Löschen eines Backups # Löschen eines Backups
borg list $1 $2 $3 --info --show-rc {{repo_url}} echo "Available Backups: "
borg list $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }}
echo "BackupName, followed by [ENTER]:" echo "BackupName, followed by [ENTER]:"
read target read target
borg delete $1 $2 $3 --info --show-rc {{repo_url}}::$target borg delete $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }}::$target
#!/bin/bash #!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
# Anzeige des Inhaltes im Borg Backup Archiv # Anzeige des Inhaltes in den Borg Backup Archiven
borg list $1 $2 $3 --info --show-rc {{repo_url}} echo "============================================="
echo "Backups Repo Info: {{ item.key }} "
BACKUPS=$(borg list $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }})
echo "$BACKUPS"
BACKUPS_LIST=$(echo "$BACKUPS" | awk '{print $1}')
for BACKUP in $BACKUPS_LIST; do
echo "============================================="
borg info $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }}::$BACKUP
done
echo "============================================="
echo "BackupName, followed by [ENTER]:"
read target
borg info $1 $2 $3 --info --show-rc {{repo_url}}::$target
#!/bin/bash #!/bin/bash
# Initialisierung des Borg Backup Archives # Initialisierung der Borg Backup Archives
# Der SSH key aus /srv/borgbackup/repo_sshkey muss vorher auf den Backupserver übertragen werden
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
if [ ! -e "/srv/borgbackup/{{ item.key }}/initialized" ]; then
echo "Initialize Repo: {{ item.key }}"
date > "/srv/borgbackup/{{ item.key }}/initialized"
borg init $1 $2 $3 --info --show-rc --encryption=repokey {{ item.value.options }} {{ item.value.repo }}
else
echo "Repo already initialized: {{ item.key }}"
fi
borg init $1 $2 $3 --info --show-rc --encryption=repokey {{repo_url}}
#!/bin/bash #!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
# Anzeige des Inhaltes in den Borg Backup Archiven
echo "===[ List Repo: {{ item.key }} ]==="
borg list $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }}
# Anzeige des Inhaltes im Borg Backup Archiv
borg list $1 $2 $3 --info --show-rc {{repo_url}}
#!/bin/bash #!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}" export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
# Einhängen eines Backups nach /mnt/ # Einhängen der Repos in /mnt/
echo "Directory to mount, followed by [ENTER]:"
read target
borg mount $1 $2 $3 --info --show-rc {{ item.value.options }} {{ item.value.repo }} $target
echo -n 'Mounting to: /mnt/borbbackupmount'
borg mount $1 $2 $3 --info --show-rc {{repo_url}} /mnt/borbbackupmount
#!/bin/bash
export BORG_PASSPHRASE="{{repo_passphrase}}"
export BORG_RSH="ssh -i /srv/borgbackup/repo_sshkey"
# Metrics output file in the prometheus node-exporter directory
PROM_FILE="/var/lib/prometheus/node-exporter/borgbackup.prom"
# Borgbackup statistiken für Prometheus erstellen
function calc_bytes {
NUM=$1
UNIT=$2
case "$UNIT" in
kB)
echo $NUM | awk '{ print $1 * 1024 }'
;;
MB)
echo $NUM | awk '{ print $1 * 1024 * 1024 }'
;;
GB)
echo $NUM | awk '{ print $1 * 1024 * 1024 * 1024 }'
;;
TB)
echo $NUM | awk '{ print $1 * 1024 * 1024 * 1024 * 1024 }'
;;
esac
}
# create temp file
TMP_FILE=$(mktemp)
echo "borgbackup_repos_count {{ borgbackup_repos|length }}" > $TMP_FILE
{% for repo in borgbackup_repos %}
BACKUPS=$(borg list {{ borgbackup_repos[repo].options }} {{ borgbackup_repos[repo].repo }})
BACKUPS_LIST=$(echo "$BACKUPS" | awk '{print $1}')
COUNTER=0
for BACKUP in $BACKUPS_LIST; do
COUNTER=$((COUNTER+1))
done
BORG_INFO=$(borg info {{ borgbackup_repos[repo].options }} {{ borgbackup_repos[repo].repo }}::$BACKUP)
echo "borgbackup_count{repo="{{ repo }}"} $COUNTER" >> $TMP_FILE
echo "borgbackup_files{repo="{{ repo }}"} $(echo "$BORG_INFO" | grep "Number of files" | awk '{print $4}')" >> $TMP_FILE
echo "borgbackup_chunks_unique{repo="{{ repo }}"} $(echo "$BORG_INFO" | grep "Chunk index" | awk '{print $3}')" >> $TMP_FILE
echo "borgbackup_chunks_total{repo="{{ repo }}"} $(echo "$BORG_INFO" | grep "Chunk index" | awk '{print $4}')" >> $TMP_FILE
# byte size calculation
LAST_SIZE=$(calc_bytes $(echo "$BORG_INFO" |grep "This archive" |awk '{print $3}') $(echo "$BORG_INFO" |grep "This archive" |awk '{print $4}'))
LAST_SIZE_COMPRESSED=$(calc_bytes $(echo "$BORG_INFO" |grep "This archive" |awk '{print $5}') $(echo "$BORG_INFO" |grep "This archive" |awk '{print $6}'))
LAST_SIZE_DEDUP=$(calc_bytes $(echo "$BORG_INFO" |grep "This archive" |awk '{print $7}') $(echo "$BORG_INFO" |grep "This archive" |awk '{print $8}'))
TOTAL_SIZE=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $3}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $4}'))
TOTAL_SIZE_COMPRESSED=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $5}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $6}'))
TOTAL_SIZE_DEDUP=$(calc_bytes $(echo "$BORG_INFO" |grep "All archives" |awk '{print $7}') $(echo "$BORG_INFO" |grep "All archives" |awk '{print $8}'))
echo "borgbackup_last_size{repo="{{ repo }}"} $LAST_SIZE" >> $TMP_FILE
echo "borgbackup_last_size_compressed{repo="{{ repo }}"} $LAST_SIZE_COMPRESSED" >> $TMP_FILE
echo "borgbackup_last_size_dedup{repo="{{ repo }}"} $LAST_SIZE_DEDUP" >> $TMP_FILE
echo "borgbackup_total_size{repo="{{ repo }}"} $TOTAL_SIZE" >> $TMP_FILE
echo "borgbackup_total_size_compressed{repo="{{ repo }}"} $TOTAL_SIZE_COMPRESSED" >> $TMP_FILE
echo "borgbackup_total_size_dedup{repo="{{ repo }}"} $TOTAL_SIZE_DEDUP" >> $TMP_FILE
{% endfor %}
# move temp file to output file
mv $TMP_FILE $PROM_FILE
chown prometheus:prometheus $PROM_FILE
echo "created BorgBackup statistic for $COUNTER backups in {{ borgbackup_repos|length }} repos: $PROM_FILE"
/var/log/borgbackup/borgbackup.log { /var/log/borgbackup/*.log {
rotate 12 rotate 12
monthly monthly
compress compress
......
---
# Pakete installieren
- name: pakete installieren
apt:
pkg: "{{ item }}"
update_cache: yes
state: installed
with_items:
- borgbackup
# User for private backups
- name: create backup user account
user:
name: "{{ item.key }}"
group: "users"
home: "/data/{{ item.key }}"
createhome: yes
with_dict: "{{ borgbackup_user }}"
- name: create authorized_keys for users 1
file:
path: "/data/{{ item.key }}/.ssh"
state: "directory"
with_dict: "{{ borgbackup_user }}"
- name: create authorized_keys for users 2
template:
src: authorized_keys
dest: "/data/{{ item.key }}/.ssh/authorized_keys"
with_dict: "{{ borgbackup_user }}"
{% for sshkey in item.value.sshkeys %}
{{ sshkey }}
{% endfor %}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment