commit 5c23ca311e6bb0a5e44caadb33fb45535da00483 Author: uzurka Date: Sat Dec 14 13:08:21 2024 +0100 Initial commit diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 0000000..bc27755 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ + +## [Unreleased] + + + +## [1.2.0] - 2022-06-15 + + +## 1.1.0 - 2022-06-15 +### Added +- Added lint and CHANGELOG CI +- borg_conf_compression variable +- Borg cron for pruning backups +- before/after check/prune/extract commands variables + +### Changed +- > to >> for borg cron jobs logs +- display correct minimal ansible version, minor template change for Python2 + +### Fix +- meta author + +### Fixed +- README.md + +### Removed +- borg_cron_purge as redondant + + +[Unreleased]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.2.0...HEAD +[1.2.0]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.1.0...1.2.0 diff --git a/CHANGELOG.tpl.md b/CHANGELOG.tpl.md new file mode 100755 index 0000000..389e978 --- /dev/null +++ b/CHANGELOG.tpl.md @@ -0,0 +1,49 @@ +{{ if .Versions -}} + +## [Unreleased] + +{{ if .Unreleased.CommitGroups -}} +{{ range .Unreleased.CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{ range .Versions }} + +## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} +{{ range .CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end -}} + +{{- if .RevertCommits -}} +### Reverts +{{ range .RevertCommits -}} +- {{ .Revert.Header }} +{{ end }} +{{ end -}} + +{{- if .NoteGroups -}} +{{ range .NoteGroups -}} +### {{ .Title }} +{{ range .Notes }} +{{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{- if .Versions }} +[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD +{{ range .Versions -}} +{{ if .Tag.Previous -}} +[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} +{{ end -}} +{{ end -}} +{{ end -}} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..6eedea9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) NoxInmortus (Alban E.G.) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100755 index 0000000..d0e5622 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# Borgbackup ![Borgbackup](https://img.shields.io/badge/Ansible-Borgbackup.svg) + +## Sommaire +* [Preview](#preview) + - [TODO](#todo) + - [Requirements](#requirements) + - [Compatibility](#compatibility) +* [Usage](#usage) + - [Variables](#variables) + - [Examples](#examples) +* [Licence](#licence) + +## Preview +Ansible role to install Borgbackup and Borgmatic. + +Set up encrypted, compressed and deduplicated backups using [Borgbackup](https://borgbackup.readthedocs.io/en/stable/) and [Borgmatic](https://github.com/witten/borgmatic). + +### TODO +- Gitlab-CI + +### Requirements +- Ansible >= 2.7 + +### Compatibility + ![Debian](https://img.shields.io/badge/Debian-Buster-blue.svg) + ![Debian](https://img.shields.io/badge/Debian-Stretch-blue.svg) + +This role has been tested only on Debian Buster & Stretch, but it should be working on every GNU/Linux distribution. + +## Usage +### Variables + +See default [variables](defaults/main.yml). + +|NAME|TYPE|REQUIRED|DEFAULT|DESCRIPTION| +|-|-|-|-|-| +|borg_encryption_passphrase|STRING|YES|Empty|Encryption passphrase| +|borg_packages|LIST|NO|See defaults|Required packages installed through local package manager| +|borg_packages_pip|LIST|NO|See defaults|Required packages installed through pip3| +|borg_conf_template|STRING|NO|`config.yaml`|Template used for main config file| +|borg_exclude_template|STRING|NO|`excludes`|Template used for exclude patterns| +|borg_user|STRING|NO|`root`|User used for borgbackups| +|borg_local_repository|STRING|NO|`/var/backups/borg`|Local repository path| +|borg_remote_repository|STRING|NO|NONE|Optional remote repository| +|borg_no_local_repository|BOOL|NO|`false`|Do not init local repository| +|borg_init_remote_repository|BOOL|NO|`false`|Init remote repository (should be set to `true` at first run only)| +|borg_encryption_type|STRING|NO|`repokey-blake2`|Encryption method, see official doc for more| +|borg_excludes_default|LIST|NO|See defaults|Defaults excluded patterns| +|borg_excludes|LIST|NO|NONE|Excludes patterns, merged with `borg_excludes_default`| +|borg_backup_dirs|LIST|NO|NONE|Folders you want to backup| +|borg_mysqldump|LIST of DICT|NO|NONE|MySQL databases, see example below| +|borg_conf_umask|STRING|NO|`0077`|Umask used when executing hooks. Defaults to the umask that borgmatic is run with| +|borg_conf_compression|STRING|NO|`lz4`|Compression algorithm used by borg. See official documentation for more details| +|borg_conf_location|DICT|NO|See defaults|Defaults options for borgmatic `location` configuration section| +|borg_conf_storage|DICT|NO|See defaults|Defaults options for borgmatic `storage` configuration section| +|borg_conf_retention_policy|DICT|NO|See defaults|Defaults options for borgmatic `retention_policy` configuration section| +|borg_conf_consistency|DICT|NO|See defaults|Defaults options for borgmatic `consistency` configuration section| +|borg_before_backup_commands|LIST|NO|NONE|Before backup commands| +|borg_after_backup_commands|LIST|NO|NONE|After backup commands| +|borg_failure_commands|LIST|NO|NONE|Failed backup commands| +|borg_before_everything_commands|LIST|NO|NONE|Before any action commands| +|borg_after_everything_commands|LIST|NO|NONE|After any action commands| +|borg_before_check_commands|LIST|NO|NONE|Before check commands| +|borg_after_check_commands|LIST|NO|NONE|After check commands| +|borg_before_prune_commands|LIST|NO|NONE|Before prune commands| +|borg_after_prune_commands|LIST|NO|NONE|After prune commands| +|borg_before_extract_commands|LIST|NO|NONE|Before extract commands| +|borg_after_extract_commands|LIST|NO|NONE|After extract commands| +|borg_cron_enable|BOOL|NO|`true`|Enable cron job| +|borg_cron_action|STRING|NO|`create`|Default borgmatic main parameter for cronjob| +|borg_cron_nice|INT|NO|`19`|Nice parameter for cron job| +|borg_cron_ionice|INT|NO|`3`|Ionice parameter for cron job| +|borg_cron_log|STRING|NO|`/var/log/borg.log`|Borg log file path| +|borg_cron|DICT|NO|See defaults|Borg cron job startup| +|borg_logrotate|BOOL|NO|`true`|Setup default Borg logrotate conf file| +|borg_scripts|BOOL|NO|`true`|Add extra scripts| + +### Examples +``` +borg_encryption_passphrase: MyS3Cr3tPa55phr4s3 +borg_backup_dirs: +- /var/www +- /home/me + +borg_cron: + hour: 23 + minute: 0 + day: '1' + weekday: '*' + month: '*' + +borg_mysqldump: + - name: all + username: root + - name: posts + hostname: database2.example.org + port: 3307 + username: root + password: trustsome1 + options: "--skip-comments" +``` + +## Sources +- https://github.com/borgbase/ansible-role-borgbackup +- https://github.com/bfabio/ansible-borg_client +- https://github.com/adhawkins/ansible-borgbase +- https://github.com/witten/borgmatic +- https://torsion.org/borgmatic/docs/reference/configuration/ +- https://borgbackup.readthedocs.io/en/stable/usage/general.html + +## Licence +MIT view [LICENSE](LICENSE) diff --git a/ansible.cfg b/ansible.cfg new file mode 100755 index 0000000..138157f --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = hosts +host_key_checking = false +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp +roles_path = roles +timeout = 10 +module_name = shell +retry_files_enabled = false +interpreter_python = /usr/bin/python3 diff --git a/config.yaml b/config.yaml new file mode 100755 index 0000000..5e78c59 --- /dev/null +++ b/config.yaml @@ -0,0 +1,50 @@ +--- +# ansible_managed /!\ +location: + source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}] + repositories: +{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %} + +{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %} + +{%+ for key,value in borg_conf_location.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +storage: +{%+ for key,value in borg_conf_storage.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +retention: +{%+ for key,value in borg_conf_retention_policy.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +consistency: +{%+ for key,value in borg_conf_consistency.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +hooks: + umask: {{ borg_conf_umask }} + before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}] + after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}] + on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}] + before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}] + after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}] + before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}] + after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}] + before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}] + after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}] + before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}] + after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}] + +{% if borg_mysqldump is defined %} + mysql_databases: +{% for db in borg_mysqldump|default([]) %} +{% for key, value in db.items()|sort %} + {% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/config.yml b/config.yml new file mode 100755 index 0000000..c9caafc --- /dev/null +++ b/config.yml @@ -0,0 +1,25 @@ +--- +- name: Create /etc/borgmatic directory + ansible.builtin.file: + state: 'directory' + path: '/etc/borgmatic' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0700' + +- name: Copy borgmatic exclude file + ansible.builtin.template: + src: '{{ borg_exclude_template }}' + dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + +- name: Copy borgmatic config file + ansible.builtin.template: + src: '{{ borg_conf_template }}' + dest: '/etc/borgmatic/{{ borg_conf_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + validate: borgmatic config validate -c %s diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100755 index 0000000..05524fe --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,82 @@ +--- +borg_packages: + - python3 + - python3-pip + - borgbackup +borg_packages_pip: + - borgmatic +borg_conf_template: config.yaml +borg_exclude_template: excludes +borg_user: root + +# Borgmatic repository +borg_local_repository: /var/backups/borg +borg_init_remote_repository: false +borg_no_local_repository: false +borg_encryption_type: 'repokey-blake2' + +# do not add /tmp !!! borgmatic uses /tmp !! +borg_excludes_default: + - '/var/tmp' + - '*.pyc' + - '/home/*/.cache' + +# Borgmatic configuration file +borg_backup_dirs: [] +borg_conf_umask: '0077' +borg_conf_compression: 'lz4' +borg_conf_location: + one_file_system: 'false' + files_cache: ctime,size,inode + exclude_from: "['/etc/borgmatic/excludes']" + exclude_caches: 'true' + exclude_if_present: .nobackup + borgmatic_source_directory: /tmp/borgmatic +borg_conf_storage: + encryption_passphrase: '{{ borg_encryption_passphrase }}' + compression: '{{ borg_conf_compression }}' + remote_rate_limit: '5000' + umask: '{{ borg_conf_umask }}' + lock_wait: '5' + archive_name_format: "'{hostname}-{now}'" + relocated_repo_access_is_ok: 'true' +borg_conf_retention_policy: + keep_within: '2d' + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 1 + keep_yearly: 1 + prefix: "'{hostname}-'" +borg_conf_consistency: + prefix: "'{hostname}-'" + check_last: 1 + checks: "['repository','extract','data']" + +borg_before_backup_commands: [] +borg_after_backup_commands: [] +borg_failure_commands: [] +borg_before_everything_commands: [] +borg_after_everything_commands: [] +borg_before_check_commands: [] +borg_after_check_commands: [] +borg_before_prune_commands: [] +borg_after_prune_commands: [] +borg_before_extract_commands: [] +borg_after_extract_commands: [] + +# Borgmatic cron variables +borg_cron_enable: true +borg_cron_action: create +borg_cron_nice: 19 +borg_cron_ionice: 3 +borg_cron_log: /var/log/borg.log +borg_cron: + hour: 23 + minute: 0 + day: '*' + weekday: '*' + month: '*' + +# Extras +borg_logrotate: true +borg_scripts: true diff --git a/excludes b/excludes new file mode 100755 index 0000000..e5b6c24 --- /dev/null +++ b/excludes @@ -0,0 +1,5 @@ +# ansible_managed /!\ + +{% for exclude in borg_excludes|default([]) + borg_excludes_default -%} +{{ exclude }} +{% endfor -%} diff --git a/extra.yml b/extra.yml new file mode 100755 index 0000000..2bf406a --- /dev/null +++ b/extra.yml @@ -0,0 +1,76 @@ +--- +- name: Set up borg cron job + ansible.builtin.cron: + name: 'Borgbackup' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: '{{ borg_cron.hour }}' + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: borg_cron_enable + +- name: Set up borg prune cron job + ansible.builtin.cron: + name: 'Borgbackup Prune' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: >- + {% if borg_cron.hour in range(0, 22) %} + {{ '%02d' | format(borg_cron.hour + 2) }} + {%- elif borg_cron.hour in [22, 23] -%} + {% if borg_cron.hour == 22 %}0{% else %}1{% endif %} + {%- endif -%} + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: + - borg_cron_enable + - borg_cron_action != 'prune' + +- name: Create borg_cron_log file + ansible.builtin.file: + state: touch + modification_time: preserve + access_time: preserve + path: '{{ borg_cron_log }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + when: borg_cron_enable + +- name: Copy s3-synchronize.sh scripts + ansible.builtin.copy: + src: files/s3-synchronize.sh + dest: /usr/local/bin/s3-synchronize.sh + mode: '0755' + when: borg_scripts + +- name: Setup logrotate + ansible.builtin.copy: + dest: /etc/logrotate.d/borg + owner: root + group: root + mode: '0644' + content: | + {{ borg_cron_log }} { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + } + when: borg_logrotate diff --git a/files/s3-synchronize.sh b/files/s3-synchronize.sh new file mode 100755 index 0000000..82acc88 --- /dev/null +++ b/files/s3-synchronize.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -euo pipefail + +#=======================================# +#-- S3 SYNCHRONIZE SCRIPT --# +#-- Author : NoxInmortus (Alban E.G.) --# +#=======================================# + +## {{{ Global variables +##------------------## +## Global Variables ## +##------------------## + +SUBJECT="s3-synchronize" +VERSION="1.0.0 (18/03/2020)" +USAGE="Usage: ${0} -hv \n +-u : upload (local to remote) \n +-d : download (remote to local) \n +-l : local path to synchronize \n +-r : remote path to synchronize (s3 url format) \n +-c : s3cmd config file (mandatory)." +HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required." +LOG="/var/log/${SUBJECT}.log" +DATE=$(date '+%F-%Hh') +## Global variables }}} +## {{{ Script variables +# SECONDS returns a count of the number of (whole) seconds the shell has been running. +startTime=${SECONDS} + +# Used to check conflict +u_arg=false +d_arg=false +## Script variables }}} + +## {{{ Option processing +##-------------------## +## Option processing ## +##-------------------## + +# If there is no arguments display ${USAGE} +if [ $# == 0 ] ; then + echo -e ${USAGE} + exit 1; +fi + +while getopts ":vhudl:r:c:" optname + do + case "${optname}" in + "v") + echo "Version ${VERSION}" + exit 0; + ;; + "h") + echo -e ${HELP} + echo -e ${USAGE} + exit 0; + ;; + "u") + sync_way="upload" + u_arg=true + ;; + "d") + sync_way="download" + d_arg=true + ;; + "l") + LOCAL=${OPTARG} + ;; + "r") + REMOTE=${OPTARG} + ;; + "c") + CONFIG_FILE=${OPTARG} + ;; + "?") + echo "Unknown option ${OPTARG}" + exit 0; + ;; + ":") + echo "No argument value for option ${OPTARG}" + exit 0; + ;; + *) + echo "Unknown error while processing options" + exit 0; + ;; + esac + done + +shift $((${OPTIND} - 1)) +## Option processing }}} + +# ----------------------------------------------------------------- +# SCRIPT LOGIC GOES HERE +# ----------------------------------------------------------------- + +## Sanity Checks +if [[ -z ${LOCAL+x} ]]; then + echo "You need to define local directory (with -l option)." + exit 1 +elif [[ -z ${REMOTE+x} ]]; then + echo "You need to define remote directory (with -r option)." + exit 1 +elif [[ -z ${CONFIG_FILE+x} ]]; then + echo "You need to define s3cmd config file (with -c option)." + exit 1 +fi + +if [ ${u_arg} == ${d_arg} ]; then + echo "You cannot use -u (upload) and -d (download) options at the same time." + exit 1 +fi + +if [ ! -d ${LOCAL} ]; then + echo "Your local path is not a directory." + exit 1 +fi + +## Lockfile +(flock -x 200 || exit 1 + +if [ ${sync_way} == "upload" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE} +elif [ ${sync_way} == "download" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL} + chmod 0700 ${LOCAL} +else + echo "Error with sync_way variable." + exit 1 +fi + +elapsedTime=$((${SECONDS} - ${startTime})) +echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec" + +)200>/var/lock/${SUBJECT}.lock + +exit 0 diff --git a/hosts b/hosts new file mode 100755 index 0000000..2302eda --- /dev/null +++ b/hosts @@ -0,0 +1 @@ +localhost ansible_connection=local diff --git a/init.yml b/init.yml new file mode 100755 index 0000000..a5eb7da --- /dev/null +++ b/init.yml @@ -0,0 +1,14 @@ +--- +- name: Init local borg repository + ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}' + args: + creates: "{{ borg_local_repository }}/data" + no_log: "{{ no_log|default('true') }}" + when: not borg_no_local_repository + +- name: Init remote borg repository + ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}' + no_log: "{{ no_log|default('true') }}" + when: + - borg_remote_repository is defined + - borg_init_remote_repository diff --git a/install.yml b/install.yml new file mode 100755 index 0000000..1170462 --- /dev/null +++ b/install.yml @@ -0,0 +1,23 @@ +--- +- name: Setup apt pref + ansible.builtin.copy: + dest: /etc/apt/preferences.d/borgbackup.pref + owner: root + group: root + mode: '0644' + content: | + # Ansible managed + Package: borgbackup + Pin: release a={{ ansible_distribution_release|lower }}-backports + Pin-Priority: 990 + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install main packages + ansible.builtin.package: + name: "{{ borg_packages }}" + +- name: Install pip3 packages + ansible.builtin.pip: + name: "{{ borg_packages_pip }}" + state: latest + executable: pip3 diff --git a/main.yml b/main.yml new file mode 100755 index 0000000..49fc73d --- /dev/null +++ b/main.yml @@ -0,0 +1,24 @@ +--- +- name: assert variables + ansible.builtin.assert: + that: borg_encryption_passphrase is defined + +- include_tasks: install.yml + tags: + - borg + - borg_install + +- include_tasks: init.yml + tags: + - borg + - borg_init + +- include_tasks: config.yml + tags: + - borg + - borg_config + +- include_tasks: extra.yml + tags: + - borg + - borg_extra diff --git a/meta/.galaxy_install_info b/meta/.galaxy_install_info new file mode 100755 index 0000000..1436591 --- /dev/null +++ b/meta/.galaxy_install_info @@ -0,0 +1,2 @@ +install_date: 'Sun 16 Jun 2024 04:56:29 PM ' +version: '' diff --git a/meta/main.yml b/meta/main.yml new file mode 100755 index 0000000..6ba181c --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,11 @@ +--- +galaxy_info: + author: NoxInmortus + license: MIT + description: Role to deploy borgbackup and borgmatic configurations. + min_ansible_version: 2.7 + platforms: + - name: Debian + galaxy_tags: + - backup +dependencies: [] diff --git a/s3-synchronize.sh b/s3-synchronize.sh new file mode 100755 index 0000000..82acc88 --- /dev/null +++ b/s3-synchronize.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -euo pipefail + +#=======================================# +#-- S3 SYNCHRONIZE SCRIPT --# +#-- Author : NoxInmortus (Alban E.G.) --# +#=======================================# + +## {{{ Global variables +##------------------## +## Global Variables ## +##------------------## + +SUBJECT="s3-synchronize" +VERSION="1.0.0 (18/03/2020)" +USAGE="Usage: ${0} -hv \n +-u : upload (local to remote) \n +-d : download (remote to local) \n +-l : local path to synchronize \n +-r : remote path to synchronize (s3 url format) \n +-c : s3cmd config file (mandatory)." +HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required." +LOG="/var/log/${SUBJECT}.log" +DATE=$(date '+%F-%Hh') +## Global variables }}} +## {{{ Script variables +# SECONDS returns a count of the number of (whole) seconds the shell has been running. +startTime=${SECONDS} + +# Used to check conflict +u_arg=false +d_arg=false +## Script variables }}} + +## {{{ Option processing +##-------------------## +## Option processing ## +##-------------------## + +# If there is no arguments display ${USAGE} +if [ $# == 0 ] ; then + echo -e ${USAGE} + exit 1; +fi + +while getopts ":vhudl:r:c:" optname + do + case "${optname}" in + "v") + echo "Version ${VERSION}" + exit 0; + ;; + "h") + echo -e ${HELP} + echo -e ${USAGE} + exit 0; + ;; + "u") + sync_way="upload" + u_arg=true + ;; + "d") + sync_way="download" + d_arg=true + ;; + "l") + LOCAL=${OPTARG} + ;; + "r") + REMOTE=${OPTARG} + ;; + "c") + CONFIG_FILE=${OPTARG} + ;; + "?") + echo "Unknown option ${OPTARG}" + exit 0; + ;; + ":") + echo "No argument value for option ${OPTARG}" + exit 0; + ;; + *) + echo "Unknown error while processing options" + exit 0; + ;; + esac + done + +shift $((${OPTIND} - 1)) +## Option processing }}} + +# ----------------------------------------------------------------- +# SCRIPT LOGIC GOES HERE +# ----------------------------------------------------------------- + +## Sanity Checks +if [[ -z ${LOCAL+x} ]]; then + echo "You need to define local directory (with -l option)." + exit 1 +elif [[ -z ${REMOTE+x} ]]; then + echo "You need to define remote directory (with -r option)." + exit 1 +elif [[ -z ${CONFIG_FILE+x} ]]; then + echo "You need to define s3cmd config file (with -c option)." + exit 1 +fi + +if [ ${u_arg} == ${d_arg} ]; then + echo "You cannot use -u (upload) and -d (download) options at the same time." + exit 1 +fi + +if [ ! -d ${LOCAL} ]; then + echo "Your local path is not a directory." + exit 1 +fi + +## Lockfile +(flock -x 200 || exit 1 + +if [ ${sync_way} == "upload" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE} +elif [ ${sync_way} == "download" ]; then + s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL} + chmod 0700 ${LOCAL} +else + echo "Error with sync_way variable." + exit 1 +fi + +elapsedTime=$((${SECONDS} - ${startTime})) +echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec" + +)200>/var/lock/${SUBJECT}.lock + +exit 0 diff --git a/tasks/config.yml b/tasks/config.yml new file mode 100755 index 0000000..c9caafc --- /dev/null +++ b/tasks/config.yml @@ -0,0 +1,25 @@ +--- +- name: Create /etc/borgmatic directory + ansible.builtin.file: + state: 'directory' + path: '/etc/borgmatic' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0700' + +- name: Copy borgmatic exclude file + ansible.builtin.template: + src: '{{ borg_exclude_template }}' + dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + +- name: Copy borgmatic config file + ansible.builtin.template: + src: '{{ borg_conf_template }}' + dest: '/etc/borgmatic/{{ borg_conf_template|basename }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + validate: borgmatic config validate -c %s diff --git a/tasks/extra.yml b/tasks/extra.yml new file mode 100755 index 0000000..2bf406a --- /dev/null +++ b/tasks/extra.yml @@ -0,0 +1,76 @@ +--- +- name: Set up borg cron job + ansible.builtin.cron: + name: 'Borgbackup' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: '{{ borg_cron.hour }}' + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: borg_cron_enable + +- name: Set up borg prune cron job + ansible.builtin.cron: + name: 'Borgbackup Prune' + cron_file: borgbackup + user: "{{ borg_user }}" + hour: >- + {% if borg_cron.hour in range(0, 22) %} + {{ '%02d' | format(borg_cron.hour + 2) }} + {%- elif borg_cron.hour in [22, 23] -%} + {% if borg_cron.hour == 22 %}0{% else %}1{% endif %} + {%- endif -%} + minute: '{{ borg_cron.minute }}' + day: "{{ borg_cron.day }}" + weekday: "{{ borg_cron.weekday }}" + month: "{{ borg_cron.month }}" + job: > + PATH=$PATH:/usr/local/bin + nice -n {{ borg_cron_nice }} + ionice -c {{ borg_cron_ionice }} + borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1 + when: + - borg_cron_enable + - borg_cron_action != 'prune' + +- name: Create borg_cron_log file + ansible.builtin.file: + state: touch + modification_time: preserve + access_time: preserve + path: '{{ borg_cron_log }}' + owner: '{{ borg_user }}' + group: '{{ borg_user }}' + mode: '0600' + when: borg_cron_enable + +- name: Copy s3-synchronize.sh scripts + ansible.builtin.copy: + src: files/s3-synchronize.sh + dest: /usr/local/bin/s3-synchronize.sh + mode: '0755' + when: borg_scripts + +- name: Setup logrotate + ansible.builtin.copy: + dest: /etc/logrotate.d/borg + owner: root + group: root + mode: '0644' + content: | + {{ borg_cron_log }} { + daily + rotate 7 + missingok + notifempty + compress + delaycompress + } + when: borg_logrotate diff --git a/tasks/init.yml b/tasks/init.yml new file mode 100755 index 0000000..a5eb7da --- /dev/null +++ b/tasks/init.yml @@ -0,0 +1,14 @@ +--- +- name: Init local borg repository + ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}' + args: + creates: "{{ borg_local_repository }}/data" + no_log: "{{ no_log|default('true') }}" + when: not borg_no_local_repository + +- name: Init remote borg repository + ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}' + no_log: "{{ no_log|default('true') }}" + when: + - borg_remote_repository is defined + - borg_init_remote_repository diff --git a/tasks/install.yml b/tasks/install.yml new file mode 100755 index 0000000..543d691 --- /dev/null +++ b/tasks/install.yml @@ -0,0 +1,24 @@ +--- +- name: Setup apt pref + ansible.builtin.copy: + dest: /etc/apt/preferences.d/borgbackup.pref + owner: root + group: root + mode: '0644' + content: | + # Ansible managed + Package: borgbackup + Pin: release a={{ ansible_distribution_release|lower }}-backports + Pin-Priority: 990 + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install main packages + ansible.builtin.package: + name: "{{ borg_packages }}" + +- name: Install pip3 packages + ansible.builtin.pip: + name: "{{ borg_packages_pip }}" + state: latest + executable: pip3 + break_system_packages: true diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100755 index 0000000..49fc73d --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: assert variables + ansible.builtin.assert: + that: borg_encryption_passphrase is defined + +- include_tasks: install.yml + tags: + - borg + - borg_install + +- include_tasks: init.yml + tags: + - borg + - borg_init + +- include_tasks: config.yml + tags: + - borg + - borg_config + +- include_tasks: extra.yml + tags: + - borg + - borg_extra diff --git a/templates/config.yaml b/templates/config.yaml new file mode 100755 index 0000000..5e78c59 --- /dev/null +++ b/templates/config.yaml @@ -0,0 +1,50 @@ +--- +# ansible_managed /!\ +location: + source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}] + repositories: +{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %} + +{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %} + +{%+ for key,value in borg_conf_location.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +storage: +{%+ for key,value in borg_conf_storage.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +retention: +{%+ for key,value in borg_conf_retention_policy.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +consistency: +{%+ for key,value in borg_conf_consistency.items()|sort %} + {{ key }}: {{ value }} +{% endfor %} + +hooks: + umask: {{ borg_conf_umask }} + before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}] + after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}] + on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}] + before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}] + after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}] + before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}] + after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}] + before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}] + after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}] + before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}] + after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}] + +{% if borg_mysqldump is defined %} + mysql_databases: +{% for db in borg_mysqldump|default([]) %} +{% for key, value in db.items()|sort %} + {% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/templates/excludes b/templates/excludes new file mode 100755 index 0000000..e5b6c24 --- /dev/null +++ b/templates/excludes @@ -0,0 +1,5 @@ +# ansible_managed /!\ + +{% for exclude in borg_excludes|default([]) + borg_excludes_default -%} +{{ exclude }} +{% endfor -%} diff --git a/tests/.ansible-lint b/tests/.ansible-lint new file mode 100755 index 0000000..3c56bbc --- /dev/null +++ b/tests/.ansible-lint @@ -0,0 +1,8 @@ +parseable: true +use_default_rules: true +verbosity: 1 +skip_list: + - role-name + - line-length + - command-instead-of-shell + - package-latest diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100755 index 0000000..138157f --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = hosts +host_key_checking = false +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp +roles_path = roles +timeout = 10 +module_name = shell +retry_files_enabled = false +interpreter_python = /usr/bin/python3 diff --git a/tests/hosts b/tests/hosts new file mode 100755 index 0000000..2302eda --- /dev/null +++ b/tests/hosts @@ -0,0 +1 @@ +localhost ansible_connection=local diff --git a/tests/playbook.yml b/tests/playbook.yml new file mode 100755 index 0000000..f8f85a9 --- /dev/null +++ b/tests/playbook.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + gather_facts: true + roles: + - {role: default}