From 5c23ca311e6bb0a5e44caadb33fb45535da00483 Mon Sep 17 00:00:00 2001 From: uzurka Date: Sat, 14 Dec 2024 13:08:21 +0100 Subject: [PATCH] Initial commit --- CHANGELOG.md | 31 +++++++++ CHANGELOG.tpl.md | 49 ++++++++++++++ LICENSE | 21 ++++++ README.md | 112 +++++++++++++++++++++++++++++++ ansible.cfg | 11 +++ config.yaml | 50 ++++++++++++++ config.yml | 25 +++++++ defaults/main.yml | 82 +++++++++++++++++++++++ excludes | 5 ++ extra.yml | 76 +++++++++++++++++++++ files/s3-synchronize.sh | 137 ++++++++++++++++++++++++++++++++++++++ hosts | 1 + init.yml | 14 ++++ install.yml | 23 +++++++ main.yml | 24 +++++++ meta/.galaxy_install_info | 2 + meta/main.yml | 11 +++ s3-synchronize.sh | 137 ++++++++++++++++++++++++++++++++++++++ tasks/config.yml | 25 +++++++ tasks/extra.yml | 76 +++++++++++++++++++++ tasks/init.yml | 14 ++++ tasks/install.yml | 24 +++++++ tasks/main.yml | 24 +++++++ templates/config.yaml | 50 ++++++++++++++ templates/excludes | 5 ++ tests/.ansible-lint | 8 +++ tests/ansible.cfg | 11 +++ tests/hosts | 1 + tests/playbook.yml | 5 ++ 29 files changed, 1054 insertions(+) create mode 100755 CHANGELOG.md create mode 100755 CHANGELOG.tpl.md create mode 100755 LICENSE create mode 100755 README.md create mode 100755 ansible.cfg create mode 100755 config.yaml create mode 100755 config.yml create mode 100755 defaults/main.yml create mode 100755 excludes create mode 100755 extra.yml create mode 100755 files/s3-synchronize.sh create mode 100755 hosts create mode 100755 init.yml create mode 100755 install.yml create mode 100755 main.yml create mode 100755 meta/.galaxy_install_info create mode 100755 meta/main.yml create mode 100755 s3-synchronize.sh create mode 100755 tasks/config.yml create mode 100755 tasks/extra.yml create mode 100755 tasks/init.yml create mode 100755 tasks/install.yml create mode 100755 tasks/main.yml create mode 100755 templates/config.yaml create mode 100755 templates/excludes create mode 100755 tests/.ansible-lint create mode 100755 tests/ansible.cfg create mode 100755 tests/hosts create mode 100755 tests/playbook.yml diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 0000000..bc27755 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ + +## [Unreleased] + + + +## [1.2.0] - 2022-06-15 + + +## 1.1.0 - 2022-06-15 +### Added +- Added lint and CHANGELOG CI +- borg_conf_compression variable +- Borg cron for pruning backups +- before/after check/prune/extract commands variables + +### Changed +- > to >> for borg cron jobs logs +- display correct minimal ansible version, minor template change for Python2 + +### Fix +- meta author + +### Fixed +- README.md + +### Removed +- borg_cron_purge as redondant + + +[Unreleased]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.2.0...HEAD +[1.2.0]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.1.0...1.2.0 diff --git a/CHANGELOG.tpl.md b/CHANGELOG.tpl.md new file mode 100755 index 0000000..389e978 --- /dev/null +++ b/CHANGELOG.tpl.md @@ -0,0 +1,49 @@ +{{ if .Versions -}} + +## [Unreleased] + +{{ if .Unreleased.CommitGroups -}} +{{ range .Unreleased.CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{ range .Versions }} + +## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} +{{ range .CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end -}} + +{{- if .RevertCommits -}} +### Reverts +{{ range .RevertCommits -}} +- {{ .Revert.Header }} +{{ end }} +{{ end -}} + +{{- if .NoteGroups -}} +{{ range .NoteGroups -}} +### {{ .Title }} +{{ range .Notes }} +{{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{- if .Versions }} +[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD +{{ range .Versions -}} +{{ if .Tag.Previous -}} +[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} +{{ end -}} +{{ end -}} +{{ end -}} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..6eedea9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) NoxInmortus (Alban E.G.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100755 index 0000000..d0e5622 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# Borgbackup ![Borgbackup](https://img.shields.io/badge/Ansible-Borgbackup.svg) + +## Sommaire +* [Preview](#preview) + - [TODO](#todo) + - [Requirements](#requirements) + - [Compatibility](#compatibility) +* [Usage](#usage) + - [Variables](#variables) + - [Examples](#examples) +* [Licence](#licence) + +## Preview +Ansible role to install Borgbackup and Borgmatic. + +Set up encrypted, compressed and deduplicated backups using [Borgbackup](https://borgbackup.readthedocs.io/en/stable/) and [Borgmatic](https://github.com/witten/borgmatic). + +### TODO +- Gitlab-CI + +### Requirements +- Ansible >= 2.7 + +### Compatibility + ![Debian](https://img.shields.io/badge/Debian-Buster-blue.svg) + ![Debian](https://img.shields.io/badge/Debian-Stretch-blue.svg) + +This role has been tested only on Debian Buster & Stretch, but it should be working on every GNU/Linux distribution. + +## Usage +### Variables + +See default [variables](defaults/main.yml). + +|NAME|TYPE|REQUIRED|DEFAULT|DESCRIPTION| +|-|-|-|-|-| +|borg_encryption_passphrase|STRING|YES|Empty|Encryption passphrase| +|borg_packages|LIST|NO|See defaults|Required packages installed through local package manager| +|borg_packages_pip|LIST|NO|See defaults|Required packages installed through pip3| +|borg_conf_template|STRING|NO|`config.yaml`|Template used for main config file| +|borg_exclude_template|STRING|NO|`excludes`|Template used for exclude patterns| +|borg_user|STRING|NO|`root`|User used for borgbackups| +|borg_local_repository|STRING|NO|`/var/backups/borg`|Local repository path| +|borg_remote_repository|STRING|NO|NONE|Optional remote repository| +|borg_no_local_repository|BOOL|NO|`false`|Do not init local repository| +|borg_init_remote_repository|BOOL|NO|`false`|Init remote repository (should be set to `true` at first run only)| +|borg_encryption_type|STRING|NO|`repokey-blake2`|Encryption method, see official doc for more| +|borg_excludes_default|LIST|NO|See defaults|Defaults excluded patterns| +|borg_excludes|LIST|NO|NONE|Excludes patterns, merged with `borg_excludes_default`| +|borg_backup_dirs|LIST|NO|NONE|Folders you want to backup| +|borg_mysqldump|LIST of DICT|NO|NONE|MySQL databases, see example below| +|borg_conf_umask|STRING|NO|`0077`|Umask used when executing hooks. Defaults to the umask that borgmatic is run with| +|borg_conf_compression|STRING|NO|`lz4`|Compression algorithm used by borg. See official documentation for more details| +|borg_conf_location|DICT|NO|See defaults|Defaults options for borgmatic `location` configuration section| +|borg_conf_storage|DICT|NO|See defaults|Defaults options for borgmatic `storage` configuration section| +|borg_conf_retention_policy|DICT|NO|See defaults|Defaults options for borgmatic `retention_policy` configuration section| +|borg_conf_consistency|DICT|NO|See defaults|Defaults options for borgmatic `consistency` configuration section| +|borg_before_backup_commands|LIST|NO|NONE|Before backup commands| +|borg_after_backup_commands|LIST|NO|NONE|After backup commands| +|borg_failure_commands|LIST|NO|NONE|Failed backup commands| +|borg_before_everything_commands|LIST|NO|NONE|Before any action commands| +|borg_after_everything_commands|LIST|NO|NONE|After any action commands| +|borg_before_check_commands|LIST|NO|NONE|Before check commands| +|borg_after_check_commands|LIST|NO|NONE|After check commands| +|borg_before_prune_commands|LIST|NO|NONE|Before prune commands| +|borg_after_prune_commands|LIST|NO|NONE|After prune commands| +|borg_before_extract_commands|LIST|NO|NONE|Before extract commands| +|borg_after_extract_commands|LIST|NO|NONE|After extract commands| +|borg_cron_enable|BOOL|NO|`true`|Enable cron job| +|borg_cron_action|STRING|NO|`create`|Default borgmatic main parameter for cronjob| +|borg_cron_nice|INT|NO|`19`|Nice parameter for cron job| +|borg_cron_ionice|INT|NO|`3`|Ionice parameter for cron job| +|borg_cron_log|STRING|NO|`/var/log/borg.log`|Borg log file path| +|borg_cron|DICT|NO|See defaults|Borg cron job startup| +|borg_logrotate|BOOL|NO|`true`|Setup default Borg logrotate conf file| +|borg_scripts|BOOL|NO|`true`|Add extra scripts| + +### Examples +``` +borg_encryption_passphrase: MyS3Cr3tPa55phr4s3 +borg_backup_dirs: +- /var/www +- /home/me + +borg_cron: + hour: 23 + minute: 0 + day: '1' + weekday: '*' + month: '*' + +borg_mysqldump: + - name: all + username: root + - name: posts + hostname: database2.example.org + port: 3307 + username: root + password: trustsome1 + options: "--skip-comments" +``` + +## Sources +- https://github.com/borgbase/ansible-role-borgbackup +- https://github.com/bfabio/ansible-borg_client +- https://github.com/adhawkins/ansible-borgbase +- https://github.com/witten/borgmatic +- https://torsion.org/borgmatic/docs/reference/configuration/ +- https://borgbackup.readthedocs.io/en/stable/usage/general.html + +## Licence +MIT view [LICENSE](LICENSE) diff --git a/ansible.cfg b/ansible.cfg new file mode 100755 index 0000000..138157f --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = hosts +host_key_checking = false +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp +roles_path = roles +timeout = 10 +module_name = shell +retry_files_enabled = false +interpreter_python = /usr/bin/python3 diff --git a/config.yaml b/config.yaml new file mode 100755 index 0000000..5e78c59 --- /dev/null +++ b/config.yaml @@ -0,0 +1,50 @@ +--- +# ansible_managed /!\ +location: + source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}] + repositories: +{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %} + +{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %} + +{%+ for key,value in borg_conf_location.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +storage: +{%+ for key,value in borg_conf_storage.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +retention: +{%+ for key,value in borg_conf_retention_policy.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +consistency: +{%+ for key,value in borg_conf_consistency.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +hooks: + umask: {{ borg_conf_umask }} + before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}] + after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}] + on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}] + before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}] + after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}] + before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}] + after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}] + before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}] + after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}] + before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}] + after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}] + +{% if borg_mysqldump is defined %} + mysql_databases: +{% for db in borg_mysqldump|default([]) %} +{% for key, value in db.items()|sort %} + {% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/config.yml b/config.yml new file mode 100755 index 0000000..c9caafc --- /dev/null +++ b/config.yml @@ -0,0 +1,25 @@ +--- +- name: Create /etc/borgmatic directory + ansible.builtin.file: + state: 'directory' + path: '/etc/borgmatic' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0700' + +- name: Copy borgmatic exclude file + ansible.builtin.template: + src: '{{ borg_exclude_template }}' + dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + +- name: Copy borgmatic config file + ansible.builtin.template: + src: '{{ borg_conf_template }}' + dest: '/etc/borgmatic/{{ borg_conf_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + validate: borgmatic config validate -c %s diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100755 index 0000000..05524fe --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,82 @@ +--- +borg_packages: + - python3 + - python3-pip + - borgbackup +borg_packages_pip: + - borgmatic +borg_conf_template: config.yaml +borg_exclude_template: excludes +borg_user: root + +# Borgmatic repository +borg_local_repository: /var/backups/borg +borg_init_remote_repository: false +borg_no_local_repository: false +borg_encryption_type: 'repokey-blake2' + +# do not add /tmp !!! borgmatic uses /tmp !! +borg_excludes_default: + - '/var/tmp' + - '*.pyc' + - '/home/*/.cache' + +# Borgmatic configuration file +borg_backup_dirs: [] +borg_conf_umask: '0077' +borg_conf_compression: 'lz4' +borg_conf_location: + one_file_system: 'false' + files_cache: ctime,size,inode + exclude_from: "['/etc/borgmatic/excludes']" + exclude_caches: 'true' + exclude_if_present: .nobackup + borgmatic_source_directory: /tmp/borgmatic +borg_conf_storage: + encryption_passphrase: '{{ borg_encryption_passphrase }}' + compression: '{{ borg_conf_compression }}' + remote_rate_limit: '5000' + umask: '{{ borg_conf_umask }}' + lock_wait: '5' + archive_name_format: "'{hostname}-{now}'" + relocated_repo_access_is_ok: 'true' +borg_conf_retention_policy: + keep_within: '2d' + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 1 + keep_yearly: 1 + prefix: "'{hostname}-'" +borg_conf_consistency: + prefix: "'{hostname}-'" + check_last: 1 + checks: "['repository','extract','data']" + +borg_before_backup_commands: [] +borg_after_backup_commands: [] +borg_failure_commands: [] +borg_before_everything_commands: [] +borg_after_everything_commands: [] +borg_before_check_commands: [] +borg_after_check_commands: [] +borg_before_prune_commands: [] +borg_after_prune_commands: [] +borg_before_extract_commands: [] +borg_after_extract_commands: [] + +# Borgmatic cron variables +borg_cron_enable: true +borg_cron_action: create +borg_cron_nice: 19 +borg_cron_ionice: 3 +borg_cron_log: /var/log/borg.log +borg_cron: + hour: 23 + minute: 0 + day: '*' + weekday: '*' + month: '*' + +# Extras +borg_logrotate: true +borg_scripts: true diff --git a/excludes b/excludes new file mode 100755 index 0000000..e5b6c24 --- /dev/null +++ b/excludes @@ -0,0 +1,5 @@ +# ansible_managed /!\ + +{% for exclude in borg_excludes|default([]) + borg_excludes_default -%} +{{ exclude }} +{% endfor -%} diff --git a/extra.yml b/extra.yml new file mode 100755 index 0000000..2bf406a --- /dev/null +++ b/extra.yml @@ -0,0 +1,76 @@ +--- +- name: Set up borg cron job + ansible.builtin.cron: + name: 'Borgbackup' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: '{{ borg_cron.hour }}' + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: borg_cron_enable + +- name: Set up borg prune cron job + ansible.builtin.cron: + name: 'Borgbackup Prune' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: >- + {% if borg_cron.hour in range(0, 22) %} + {{ '%02d' | format(borg_cron.hour + 2) }} + {%- elif borg_cron.hour in [22, 23] -%} + {% if borg_cron.hour == 22 %}0{% else %}1{% endif %} + {%- endif -%} + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: + - borg_cron_enable + - borg_cron_action != 'prune' + +- name: Create borg_cron_log file + ansible.builtin.file: + state: touch + modification_time: preserve + access_time: preserve + path: '{{ borg_cron_log }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + when: borg_cron_enable + +- name: Copy s3-synchronize.sh scripts + ansible.builtin.copy: + src: files/s3-synchronize.sh + dest: /usr/local/bin/s3-synchronize.sh + mode: '0755' + when: borg_scripts + +- name: Setup logrotate + ansible.builtin.copy: + dest: /etc/logrotate.d/borg + owner: root + group: root + mode: '0644' + content: | + {{ borg_cron_log }} { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + } + when: borg_logrotate diff --git a/files/s3-synchronize.sh b/files/s3-synchronize.sh new file mode 100755 index 0000000..82acc88 --- /dev/null +++ b/files/s3-synchronize.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -euo pipefail + +#=======================================# +#-- S3 SYNCHRONIZE SCRIPT --# +#-- Author : NoxInmortus (Alban E.G.) --# +#=======================================# + +## {{{ Global variables +##------------------## +## Global Variables ## +##------------------## + +SUBJECT="s3-synchronize" +VERSION="1.0.0 (18/03/2020)" +USAGE="Usage: ${0} -hv \n +-u : upload (local to remote) \n +-d : download (remote to local) \n +-l : local path to synchronize \n +-r : remote path to synchronize (s3 url format) \n +-c : s3cmd config file (mandatory)." +HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required." +LOG="/var/log/${SUBJECT}.log" +DATE=$(date '+%F-%Hh') +## Global variables }}} +## {{{ Script variables +# SECONDS returns a count of the number of (whole) seconds the shell has been running. +startTime=${SECONDS} + +# Used to check conflict +u_arg=false +d_arg=false +## Script variables }}} + +## {{{ Option processing +##-------------------## +## Option processing ## +##-------------------## + +# If there is no arguments display ${USAGE} +if [ $# == 0 ] ; then + echo -e ${USAGE} + exit 1; +fi + +while getopts ":vhudl:r:c:" optname + do + case "${optname}" in + "v") + echo "Version ${VERSION}" + exit 0; + ;; + "h") + echo -e ${HELP} + echo -e ${USAGE} + exit 0; + ;; + "u") + sync_way="upload" + u_arg=true + ;; + "d") + sync_way="download" + d_arg=true + ;; + "l") + LOCAL=${OPTARG} + ;; + "r") + REMOTE=${OPTARG} + ;; + "c") + CONFIG_FILE=${OPTARG} + ;; + "?") + echo "Unknown option ${OPTARG}" + exit 0; + ;; + ":") + echo "No argument value for option ${OPTARG}" + exit 0; + ;; + *) + echo "Unknown error while processing options" + exit 0; + ;; + esac + done + +shift $((${OPTIND} - 1)) +## Option processing }}} + +# ----------------------------------------------------------------- +# SCRIPT LOGIC GOES HERE +# ----------------------------------------------------------------- + +## Sanity Checks +if [[ -z ${LOCAL+x} ]]; then + echo "You need to define local directory (with -l option)." + exit 1 +elif [[ -z ${REMOTE+x} ]]; then + echo "You need to define remote directory (with -r option)." + exit 1 +elif [[ -z ${CONFIG_FILE+x} ]]; then + echo "You need to define s3cmd config file (with -c option)." + exit 1 +fi + +if [ ${u_arg} == ${d_arg} ]; then + echo "You cannot use -u (upload) and -d (download) options at the same time." + exit 1 +fi + +if [ ! -d ${LOCAL} ]; then + echo "Your local path is not a directory." + exit 1 +fi + +## Lockfile +(flock -x 200 || exit 1 + +if [ ${sync_way} == "upload" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE} +elif [ ${sync_way} == "download" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL} + chmod 0700 ${LOCAL} +else + echo "Error with sync_way variable." + exit 1 +fi + +elapsedTime=$((${SECONDS} - ${startTime})) +echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec" + +)200>/var/lock/${SUBJECT}.lock + +exit 0 diff --git a/hosts b/hosts new file mode 100755 index 0000000..2302eda --- /dev/null +++ b/hosts @@ -0,0 +1 @@ +localhost ansible_connection=local diff --git a/init.yml b/init.yml new file mode 100755 index 0000000..a5eb7da --- /dev/null +++ b/init.yml @@ -0,0 +1,14 @@ +--- +- name: Init local borg repository + ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}' + args: + creates: "{{ borg_local_repository }}/data" + no_log: "{{ no_log|default('true') }}" + when: not borg_no_local_repository + +- name: Init remote borg repository + ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}' + no_log: "{{ no_log|default('true') }}" + when: + - borg_remote_repository is defined + - borg_init_remote_repository diff --git a/install.yml b/install.yml new file mode 100755 index 0000000..1170462 --- /dev/null +++ b/install.yml @@ -0,0 +1,23 @@ +--- +- name: Setup apt pref + ansible.builtin.copy: + dest: /etc/apt/preferences.d/borgbackup.pref + owner: root + group: root + mode: '0644' + content: | + # Ansible managed + Package: borgbackup + Pin: release a={{ ansible_distribution_release|lower }}-backports + Pin-Priority: 990 + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install main packages + ansible.builtin.package: + name: "{{ borg_packages }}" + +- name: Install pip3 packages + ansible.builtin.pip: + name: "{{ borg_packages_pip }}" + state: latest + executable: pip3 diff --git a/main.yml b/main.yml new file mode 100755 index 0000000..49fc73d --- /dev/null +++ b/main.yml @@ -0,0 +1,24 @@ +--- +- name: assert variables + ansible.builtin.assert: + that: borg_encryption_passphrase is defined + +- include_tasks: install.yml + tags: + - borg + - borg_install + +- include_tasks: init.yml + tags: + - borg + - borg_init + +- include_tasks: config.yml + tags: + - borg + - borg_config + +- include_tasks: extra.yml + tags: + - borg + - borg_extra diff --git a/meta/.galaxy_install_info b/meta/.galaxy_install_info new file mode 100755 index 0000000..1436591 --- /dev/null +++ b/meta/.galaxy_install_info @@ -0,0 +1,2 @@ +install_date: 'Sun 16 Jun 2024 04:56:29 PM ' +version: '' diff --git a/meta/main.yml b/meta/main.yml new file mode 100755 index 0000000..6ba181c --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,11 @@ +--- +galaxy_info: + author: NoxInmortus + license: MIT + description: Role to deploy borgbackup and borgmatic configurations. + min_ansible_version: 2.7 + platforms: + - name: Debian + galaxy_tags: + - backup +dependencies: [] diff --git a/s3-synchronize.sh b/s3-synchronize.sh new file mode 100755 index 0000000..82acc88 --- /dev/null +++ b/s3-synchronize.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -euo pipefail + +#=======================================# +#-- S3 SYNCHRONIZE SCRIPT --# +#-- Author : NoxInmortus (Alban E.G.) --# +#=======================================# + +## {{{ Global variables +##------------------## +## Global Variables ## +##------------------## + +SUBJECT="s3-synchronize" +VERSION="1.0.0 (18/03/2020)" +USAGE="Usage: ${0} -hv \n +-u : upload (local to remote) \n +-d : download (remote to local) \n +-l : local path to synchronize \n +-r : remote path to synchronize (s3 url format) \n +-c : s3cmd config file (mandatory)." +HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required." +LOG="/var/log/${SUBJECT}.log" +DATE=$(date '+%F-%Hh') +## Global variables }}} +## {{{ Script variables +# SECONDS returns a count of the number of (whole) seconds the shell has been running. +startTime=${SECONDS} + +# Used to check conflict +u_arg=false +d_arg=false +## Script variables }}} + +## {{{ Option processing +##-------------------## +## Option processing ## +##-------------------## + +# If there is no arguments display ${USAGE} +if [ $# == 0 ] ; then + echo -e ${USAGE} + exit 1; +fi + +while getopts ":vhudl:r:c:" optname + do + case "${optname}" in + "v") + echo "Version ${VERSION}" + exit 0; + ;; + "h") + echo -e ${HELP} + echo -e ${USAGE} + exit 0; + ;; + "u") + sync_way="upload" + u_arg=true + ;; + "d") + sync_way="download" + d_arg=true + ;; + "l") + LOCAL=${OPTARG} + ;; + "r") + REMOTE=${OPTARG} + ;; + "c") + CONFIG_FILE=${OPTARG} + ;; + "?") + echo "Unknown option ${OPTARG}" + exit 0; + ;; + ":") + echo "No argument value for option ${OPTARG}" + exit 0; + ;; + *) + echo "Unknown error while processing options" + exit 0; + ;; + esac + done + +shift $((${OPTIND} - 1)) +## Option processing }}} + +# ----------------------------------------------------------------- +# SCRIPT LOGIC GOES HERE +# ----------------------------------------------------------------- + +## Sanity Checks +if [[ -z ${LOCAL+x} ]]; then + echo "You need to define local directory (with -l option)." + exit 1 +elif [[ -z ${REMOTE+x} ]]; then + echo "You need to define remote directory (with -r option)." + exit 1 +elif [[ -z ${CONFIG_FILE+x} ]]; then + echo "You need to define s3cmd config file (with -c option)." + exit 1 +fi + +if [ ${u_arg} == ${d_arg} ]; then + echo "You cannot use -u (upload) and -d (download) options at the same time." + exit 1 +fi + +if [ ! -d ${LOCAL} ]; then + echo "Your local path is not a directory." + exit 1 +fi + +## Lockfile +(flock -x 200 || exit 1 + +if [ ${sync_way} == "upload" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE} +elif [ ${sync_way} == "download" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL} + chmod 0700 ${LOCAL} +else + echo "Error with sync_way variable." + exit 1 +fi + +elapsedTime=$((${SECONDS} - ${startTime})) +echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec" + +)200>/var/lock/${SUBJECT}.lock + +exit 0 diff --git a/tasks/config.yml b/tasks/config.yml new file mode 100755 index 0000000..c9caafc --- /dev/null +++ b/tasks/config.yml @@ -0,0 +1,25 @@ +--- +- name: Create /etc/borgmatic directory + ansible.builtin.file: + state: 'directory' + path: '/etc/borgmatic' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0700' + +- name: Copy borgmatic exclude file + ansible.builtin.template: + src: '{{ borg_exclude_template }}' + dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + +- name: Copy borgmatic config file + ansible.builtin.template: + src: '{{ borg_conf_template }}' + dest: '/etc/borgmatic/{{ borg_conf_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + validate: borgmatic config validate -c %s diff --git a/tasks/extra.yml b/tasks/extra.yml new file mode 100755 index 0000000..2bf406a --- /dev/null +++ b/tasks/extra.yml @@ -0,0 +1,76 @@ +--- +- name: Set up borg cron job + ansible.builtin.cron: + name: 'Borgbackup' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: '{{ borg_cron.hour }}' + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: borg_cron_enable + +- name: Set up borg prune cron job + ansible.builtin.cron: + name: 'Borgbackup Prune' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: >- + {% if borg_cron.hour in range(0, 22) %} + {{ '%02d' | format(borg_cron.hour + 2) }} + {%- elif borg_cron.hour in [22, 23] -%} + {% if borg_cron.hour == 22 %}0{% else %}1{% endif %} + {%- endif -%} + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: + - borg_cron_enable + - borg_cron_action != 'prune' + +- name: Create borg_cron_log file + ansible.builtin.file: + state: touch + modification_time: preserve + access_time: preserve + path: '{{ borg_cron_log }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + when: borg_cron_enable + +- name: Copy s3-synchronize.sh scripts + ansible.builtin.copy: + src: files/s3-synchronize.sh + dest: /usr/local/bin/s3-synchronize.sh + mode: '0755' + when: borg_scripts + +- name: Setup logrotate + ansible.builtin.copy: + dest: /etc/logrotate.d/borg + owner: root + group: root + mode: '0644' + content: | + {{ borg_cron_log }} { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + } + when: borg_logrotate diff --git a/tasks/init.yml b/tasks/init.yml new file mode 100755 index 0000000..a5eb7da --- /dev/null +++ b/tasks/init.yml @@ -0,0 +1,14 @@ +--- +- name: Init local borg repository + ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}' + args: + creates: "{{ borg_local_repository }}/data" + no_log: "{{ no_log|default('true') }}" + when: not borg_no_local_repository + +- name: Init remote borg repository + ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}' + no_log: "{{ no_log|default('true') }}" + when: + - borg_remote_repository is defined + - borg_init_remote_repository diff --git a/tasks/install.yml b/tasks/install.yml new file mode 100755 index 0000000..543d691 --- /dev/null +++ b/tasks/install.yml @@ -0,0 +1,24 @@ +--- +- name: Setup apt pref + ansible.builtin.copy: + dest: /etc/apt/preferences.d/borgbackup.pref + owner: root + group: root + mode: '0644' + content: | + # Ansible managed + Package: borgbackup + Pin: release a={{ ansible_distribution_release|lower }}-backports + Pin-Priority: 990 + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install main packages + ansible.builtin.package: + name: "{{ borg_packages }}" + +- name: Install pip3 packages + ansible.builtin.pip: + name: "{{ borg_packages_pip }}" + state: latest + executable: pip3 + break_system_packages: true diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100755 index 0000000..49fc73d --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: assert variables + ansible.builtin.assert: + that: borg_encryption_passphrase is defined + +- include_tasks: install.yml + tags: + - borg + - borg_install + +- include_tasks: init.yml + tags: + - borg + - borg_init + +- include_tasks: config.yml + tags: + - borg + - borg_config + +- include_tasks: extra.yml + tags: + - borg + - borg_extra diff --git a/templates/config.yaml b/templates/config.yaml new file mode 100755 index 0000000..5e78c59 --- /dev/null +++ b/templates/config.yaml @@ -0,0 +1,50 @@ +--- +# ansible_managed /!\ +location: + source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}] + repositories: +{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %} + +{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %} + +{%+ for key,value in borg_conf_location.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +storage: +{%+ for key,value in borg_conf_storage.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +retention: +{%+ for key,value in borg_conf_retention_policy.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +consistency: +{%+ for key,value in borg_conf_consistency.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +hooks: + umask: {{ borg_conf_umask }} + before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}] + after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}] + on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}] + before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}] + after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}] + before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}] + after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}] + before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}] + after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}] + before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}] + after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}] + +{% if borg_mysqldump is defined %} + mysql_databases: +{% for db in borg_mysqldump|default([]) %} +{% for key, value in db.items()|sort %} + {% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/templates/excludes b/templates/excludes new file mode 100755 index 0000000..e5b6c24 --- /dev/null +++ b/templates/excludes @@ -0,0 +1,5 @@ +# ansible_managed /!\ + +{% for exclude in borg_excludes|default([]) + borg_excludes_default -%} +{{ exclude }} +{% endfor -%} diff --git a/tests/.ansible-lint b/tests/.ansible-lint new file mode 100755 index 0000000..3c56bbc --- /dev/null +++ b/tests/.ansible-lint @@ -0,0 +1,8 @@ +parseable: true +use_default_rules: true +verbosity: 1 +skip_list: + - role-name + - line-length + - command-instead-of-shell + - package-latest diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100755 index 0000000..138157f --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = hosts +host_key_checking = false +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp +roles_path = roles +timeout = 10 +module_name = shell +retry_files_enabled = false +interpreter_python = /usr/bin/python3 diff --git a/tests/hosts b/tests/hosts new file mode 100755 index 0000000..2302eda --- /dev/null +++ b/tests/hosts @@ -0,0 +1 @@ +localhost ansible_connection=local diff --git a/tests/playbook.yml b/tests/playbook.yml new file mode 100755 index 0000000..f8f85a9 --- /dev/null +++ b/tests/playbook.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + gather_facts: true + roles: + - {role: default}