Initial commit

This commit is contained in:
2024-12-14 13:08:21 +01:00
commit 5c23ca311e
29 changed files with 1054 additions and 0 deletions

31
CHANGELOG.md Executable file
View File

@ -0,0 +1,31 @@
<a name="unreleased"></a>
## [Unreleased]
<a name="1.2.0"></a>
## [1.2.0] - 2022-06-15
<a name="1.1.0"></a>
## 1.1.0 - 2022-06-15
### Added
- Added lint and CHANGELOG CI
- borg_conf_compression variable
- Borg cron for pruning backups
- before/after check/prune/extract commands variables
### Changed
- > to >> for borg cron jobs logs
- display correct minimal ansible version, minor template change for Python2
### Fix
- meta author
### Fixed
- README.md
### Removed
- borg_cron_purge as redondant
[Unreleased]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.2.0...HEAD
[1.2.0]: https://git.tools01.noxinmortus.fr/sysadmins/ansible/role-borgbackup/compare/1.1.0...1.2.0

49
CHANGELOG.tpl.md Executable file
View File

@ -0,0 +1,49 @@
{{ if .Versions -}}
<a name="unreleased"></a>
## [Unreleased]
{{ if .Unreleased.CommitGroups -}}
{{ range .Unreleased.CommitGroups -}}
### {{ .Title }}
{{ range .Commits -}}
- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
{{ end }}
{{ end -}}
{{ end -}}
{{ end -}}
{{ range .Versions }}
<a name="{{ .Tag.Name }}"></a>
## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }}
{{ range .CommitGroups -}}
### {{ .Title }}
{{ range .Commits -}}
- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
{{ end }}
{{ end -}}
{{- if .RevertCommits -}}
### Reverts
{{ range .RevertCommits -}}
- {{ .Revert.Header }}
{{ end }}
{{ end -}}
{{- if .NoteGroups -}}
{{ range .NoteGroups -}}
### {{ .Title }}
{{ range .Notes }}
{{ .Body }}
{{ end }}
{{ end -}}
{{ end -}}
{{ end -}}
{{- if .Versions }}
[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
{{ range .Versions -}}
{{ if .Tag.Previous -}}
[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
{{ end -}}
{{ end -}}
{{ end -}}

21
LICENSE Executable file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) NoxInmortus (Alban E.G.)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

112
README.md Executable file
View File

@ -0,0 +1,112 @@
# Borgbackup ![Borgbackup](https://img.shields.io/badge/Ansible-Borgbackup.svg)
## Sommaire
* [Preview](#preview)
- [TODO](#todo)
- [Requirements](#requirements)
- [Compatibility](#compatibility)
* [Usage](#usage)
- [Variables](#variables)
- [Examples](#examples)
* [Licence](#licence)
## Preview
Ansible role to install Borgbackup and Borgmatic.
Set up encrypted, compressed and deduplicated backups using [Borgbackup](https://borgbackup.readthedocs.io/en/stable/) and [Borgmatic](https://github.com/witten/borgmatic).
### TODO
- Gitlab-CI
### Requirements
- Ansible >= 2.7
### Compatibility
![Debian](https://img.shields.io/badge/Debian-Buster-blue.svg)
![Debian](https://img.shields.io/badge/Debian-Stretch-blue.svg)
This role has been tested only on Debian Buster & Stretch, but it should be working on every GNU/Linux distribution.
## Usage
### Variables
See default [variables](defaults/main.yml).
|NAME|TYPE|REQUIRED|DEFAULT|DESCRIPTION|
|-|-|-|-|-|
|borg_encryption_passphrase|STRING|YES|Empty|Encryption passphrase|
|borg_packages|LIST|NO|See defaults|Required packages installed through local package manager|
|borg_packages_pip|LIST|NO|See defaults|Required packages installed through pip3|
|borg_conf_template|STRING|NO|`config.yaml`|Template used for main config file|
|borg_exclude_template|STRING|NO|`excludes`|Template used for exclude patterns|
|borg_user|STRING|NO|`root`|User used for borgbackups|
|borg_local_repository|STRING|NO|`/var/backups/borg`|Local repository path|
|borg_remote_repository|STRING|NO|NONE|Optional remote repository|
|borg_no_local_repository|BOOL|NO|`false`|Do not init local repository|
|borg_init_remote_repository|BOOL|NO|`false`|Init remote repository (should be set to `true` at first run only)|
|borg_encryption_type|STRING|NO|`repokey-blake2`|Encryption method, see official doc for more|
|borg_excludes_default|LIST|NO|See defaults|Defaults excluded patterns|
|borg_excludes|LIST|NO|NONE|Excludes patterns, merged with `borg_excludes_default`|
|borg_backup_dirs|LIST|NO|NONE|Folders you want to backup|
|borg_mysqldump|LIST of DICT|NO|NONE|MySQL databases, see example below|
|borg_conf_umask|STRING|NO|`0077`|Umask used when executing hooks. Defaults to the umask that borgmatic is run with|
|borg_conf_compression|STRING|NO|`lz4`|Compression algorithm used by borg. See official documentation for more details|
|borg_conf_location|DICT|NO|See defaults|Defaults options for borgmatic `location` configuration section|
|borg_conf_storage|DICT|NO|See defaults|Defaults options for borgmatic `storage` configuration section|
|borg_conf_retention_policy|DICT|NO|See defaults|Defaults options for borgmatic `retention_policy` configuration section|
|borg_conf_consistency|DICT|NO|See defaults|Defaults options for borgmatic `consistency` configuration section|
|borg_before_backup_commands|LIST|NO|NONE|Before backup commands|
|borg_after_backup_commands|LIST|NO|NONE|After backup commands|
|borg_failure_commands|LIST|NO|NONE|Failed backup commands|
|borg_before_everything_commands|LIST|NO|NONE|Before any action commands|
|borg_after_everything_commands|LIST|NO|NONE|After any action commands|
|borg_before_check_commands|LIST|NO|NONE|Before check commands|
|borg_after_check_commands|LIST|NO|NONE|After check commands|
|borg_before_prune_commands|LIST|NO|NONE|Before prune commands|
|borg_after_prune_commands|LIST|NO|NONE|After prune commands|
|borg_before_extract_commands|LIST|NO|NONE|Before extract commands|
|borg_after_extract_commands|LIST|NO|NONE|After extract commands|
|borg_cron_enable|BOOL|NO|`true`|Enable cron job|
|borg_cron_action|STRING|NO|`create`|Default borgmatic main parameter for cronjob|
|borg_cron_nice|INT|NO|`19`|Nice parameter for cron job|
|borg_cron_ionice|INT|NO|`3`|Ionice parameter for cron job|
|borg_cron_log|STRING|NO|`/var/log/borg.log`|Borg log file path|
|borg_cron|DICT|NO|See defaults|Borg cron job startup|
|borg_logrotate|BOOL|NO|`true`|Setup default Borg logrotate conf file|
|borg_scripts|BOOL|NO|`true`|Add extra scripts|
### Examples
```
borg_encryption_passphrase: MyS3Cr3tPa55phr4s3
borg_backup_dirs:
- /var/www
- /home/me
borg_cron:
hour: 23
minute: 0
day: '1'
weekday: '*'
month: '*'
borg_mysqldump:
- name: all
username: root
- name: posts
hostname: database2.example.org
port: 3307
username: root
password: trustsome1
options: "--skip-comments"
```
## Sources
- https://github.com/borgbase/ansible-role-borgbackup
- https://github.com/bfabio/ansible-borg_client
- https://github.com/adhawkins/ansible-borgbase
- https://github.com/witten/borgmatic
- https://torsion.org/borgmatic/docs/reference/configuration/
- https://borgbackup.readthedocs.io/en/stable/usage/general.html
## Licence
MIT view [LICENSE](LICENSE)

11
ansible.cfg Executable file
View File

@ -0,0 +1,11 @@
[defaults]
inventory = hosts
host_key_checking = false
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /tmp
roles_path = roles
timeout = 10
module_name = shell
retry_files_enabled = false
interpreter_python = /usr/bin/python3

50
config.yaml Executable file
View File

@ -0,0 +1,50 @@
---
# ansible_managed /!\
location:
source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}]
repositories:
{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %}
{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %}
{%+ for key,value in borg_conf_location.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
storage:
{%+ for key,value in borg_conf_storage.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
retention:
{%+ for key,value in borg_conf_retention_policy.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
consistency:
{%+ for key,value in borg_conf_consistency.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
hooks:
umask: {{ borg_conf_umask }}
before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}]
after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}]
on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}]
before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}]
after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}]
before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}]
after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}]
before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}]
after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}]
before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}]
after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}]
{% if borg_mysqldump is defined %}
mysql_databases:
{% for db in borg_mysqldump|default([]) %}
{% for key, value in db.items()|sort %}
{% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }}
{% endfor %}
{% endfor %}
{% endif %}

25
config.yml Executable file
View File

@ -0,0 +1,25 @@
---
- name: Create /etc/borgmatic directory
ansible.builtin.file:
state: 'directory'
path: '/etc/borgmatic'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0700'
- name: Copy borgmatic exclude file
ansible.builtin.template:
src: '{{ borg_exclude_template }}'
dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
- name: Copy borgmatic config file
ansible.builtin.template:
src: '{{ borg_conf_template }}'
dest: '/etc/borgmatic/{{ borg_conf_template|basename }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
validate: borgmatic config validate -c %s

82
defaults/main.yml Executable file
View File

@ -0,0 +1,82 @@
---
borg_packages:
- python3
- python3-pip
- borgbackup
borg_packages_pip:
- borgmatic
borg_conf_template: config.yaml
borg_exclude_template: excludes
borg_user: root
# Borgmatic repository
borg_local_repository: /var/backups/borg
borg_init_remote_repository: false
borg_no_local_repository: false
borg_encryption_type: 'repokey-blake2'
# do not add /tmp !!! borgmatic uses /tmp !!
borg_excludes_default:
- '/var/tmp'
- '*.pyc'
- '/home/*/.cache'
# Borgmatic configuration file
borg_backup_dirs: []
borg_conf_umask: '0077'
borg_conf_compression: 'lz4'
borg_conf_location:
one_file_system: 'false'
files_cache: ctime,size,inode
exclude_from: "['/etc/borgmatic/excludes']"
exclude_caches: 'true'
exclude_if_present: .nobackup
borgmatic_source_directory: /tmp/borgmatic
borg_conf_storage:
encryption_passphrase: '{{ borg_encryption_passphrase }}'
compression: '{{ borg_conf_compression }}'
remote_rate_limit: '5000'
umask: '{{ borg_conf_umask }}'
lock_wait: '5'
archive_name_format: "'{hostname}-{now}'"
relocated_repo_access_is_ok: 'true'
borg_conf_retention_policy:
keep_within: '2d'
keep_daily: 7
keep_weekly: 4
keep_monthly: 1
keep_yearly: 1
prefix: "'{hostname}-'"
borg_conf_consistency:
prefix: "'{hostname}-'"
check_last: 1
checks: "['repository','extract','data']"
borg_before_backup_commands: []
borg_after_backup_commands: []
borg_failure_commands: []
borg_before_everything_commands: []
borg_after_everything_commands: []
borg_before_check_commands: []
borg_after_check_commands: []
borg_before_prune_commands: []
borg_after_prune_commands: []
borg_before_extract_commands: []
borg_after_extract_commands: []
# Borgmatic cron variables
borg_cron_enable: true
borg_cron_action: create
borg_cron_nice: 19
borg_cron_ionice: 3
borg_cron_log: /var/log/borg.log
borg_cron:
hour: 23
minute: 0
day: '*'
weekday: '*'
month: '*'
# Extras
borg_logrotate: true
borg_scripts: true

5
excludes Executable file
View File

@ -0,0 +1,5 @@
# ansible_managed /!\
{% for exclude in borg_excludes|default([]) + borg_excludes_default -%}
{{ exclude }}
{% endfor -%}

76
extra.yml Executable file
View File

@ -0,0 +1,76 @@
---
- name: Set up borg cron job
ansible.builtin.cron:
name: 'Borgbackup'
cron_file: borgbackup
user: "{{ borg_user }}"
hour: '{{ borg_cron.hour }}'
minute: '{{ borg_cron.minute }}'
day: "{{ borg_cron.day }}"
weekday: "{{ borg_cron.weekday }}"
month: "{{ borg_cron.month }}"
job: >
PATH=$PATH:/usr/local/bin
nice -n {{ borg_cron_nice }}
ionice -c {{ borg_cron_ionice }}
borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1
when: borg_cron_enable
- name: Set up borg prune cron job
ansible.builtin.cron:
name: 'Borgbackup Prune'
cron_file: borgbackup
user: "{{ borg_user }}"
hour: >-
{% if borg_cron.hour in range(0, 22) %}
{{ '%02d' | format(borg_cron.hour + 2) }}
{%- elif borg_cron.hour in [22, 23] -%}
{% if borg_cron.hour == 22 %}0{% else %}1{% endif %}
{%- endif -%}
minute: '{{ borg_cron.minute }}'
day: "{{ borg_cron.day }}"
weekday: "{{ borg_cron.weekday }}"
month: "{{ borg_cron.month }}"
job: >
PATH=$PATH:/usr/local/bin
nice -n {{ borg_cron_nice }}
ionice -c {{ borg_cron_ionice }}
borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1
when:
- borg_cron_enable
- borg_cron_action != 'prune'
- name: Create borg_cron_log file
ansible.builtin.file:
state: touch
modification_time: preserve
access_time: preserve
path: '{{ borg_cron_log }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
when: borg_cron_enable
- name: Copy s3-synchronize.sh scripts
ansible.builtin.copy:
src: files/s3-synchronize.sh
dest: /usr/local/bin/s3-synchronize.sh
mode: '0755'
when: borg_scripts
- name: Setup logrotate
ansible.builtin.copy:
dest: /etc/logrotate.d/borg
owner: root
group: root
mode: '0644'
content: |
{{ borg_cron_log }} {
daily
rotate 7
missingok
notifempty
compress
delaycompress
}
when: borg_logrotate

137
files/s3-synchronize.sh Executable file
View File

@ -0,0 +1,137 @@
#!/usr/bin/env bash
set -euo pipefail
#=======================================#
#-- S3 SYNCHRONIZE SCRIPT --#
#-- Author : NoxInmortus (Alban E.G.) --#
#=======================================#
## {{{ Global variables
##------------------##
## Global Variables ##
##------------------##
SUBJECT="s3-synchronize"
VERSION="1.0.0 (18/03/2020)"
USAGE="Usage: ${0} -hv \n
-u : upload (local to remote) \n
-d : download (remote to local) \n
-l : local path to synchronize \n
-r : remote path to synchronize (s3 url format) \n
-c : s3cmd config file (mandatory)."
HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required."
LOG="/var/log/${SUBJECT}.log"
DATE=$(date '+%F-%Hh')
## Global variables }}}
## {{{ Script variables
# SECONDS returns a count of the number of (whole) seconds the shell has been running.
startTime=${SECONDS}
# Used to check conflict
u_arg=false
d_arg=false
## Script variables }}}
## {{{ Option processing
##-------------------##
## Option processing ##
##-------------------##
# If there is no arguments display ${USAGE}
if [ $# == 0 ] ; then
echo -e ${USAGE}
exit 1;
fi
while getopts ":vhudl:r:c:" optname
do
case "${optname}" in
"v")
echo "Version ${VERSION}"
exit 0;
;;
"h")
echo -e ${HELP}
echo -e ${USAGE}
exit 0;
;;
"u")
sync_way="upload"
u_arg=true
;;
"d")
sync_way="download"
d_arg=true
;;
"l")
LOCAL=${OPTARG}
;;
"r")
REMOTE=${OPTARG}
;;
"c")
CONFIG_FILE=${OPTARG}
;;
"?")
echo "Unknown option ${OPTARG}"
exit 0;
;;
":")
echo "No argument value for option ${OPTARG}"
exit 0;
;;
*)
echo "Unknown error while processing options"
exit 0;
;;
esac
done
shift $((${OPTIND} - 1))
## Option processing }}}
# -----------------------------------------------------------------
# SCRIPT LOGIC GOES HERE
# -----------------------------------------------------------------
## Sanity Checks
if [[ -z ${LOCAL+x} ]]; then
echo "You need to define local directory (with -l option)."
exit 1
elif [[ -z ${REMOTE+x} ]]; then
echo "You need to define remote directory (with -r option)."
exit 1
elif [[ -z ${CONFIG_FILE+x} ]]; then
echo "You need to define s3cmd config file (with -c option)."
exit 1
fi
if [ ${u_arg} == ${d_arg} ]; then
echo "You cannot use -u (upload) and -d (download) options at the same time."
exit 1
fi
if [ ! -d ${LOCAL} ]; then
echo "Your local path is not a directory."
exit 1
fi
## Lockfile
(flock -x 200 || exit 1
if [ ${sync_way} == "upload" ]; then
s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE}
elif [ ${sync_way} == "download" ]; then
s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL}
chmod 0700 ${LOCAL}
else
echo "Error with sync_way variable."
exit 1
fi
elapsedTime=$((${SECONDS} - ${startTime}))
echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec"
)200>/var/lock/${SUBJECT}.lock
exit 0

1
hosts Executable file
View File

@ -0,0 +1 @@
localhost ansible_connection=local

14
init.yml Executable file
View File

@ -0,0 +1,14 @@
---
- name: Init local borg repository
ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}'
args:
creates: "{{ borg_local_repository }}/data"
no_log: "{{ no_log|default('true') }}"
when: not borg_no_local_repository
- name: Init remote borg repository
ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}'
no_log: "{{ no_log|default('true') }}"
when:
- borg_remote_repository is defined
- borg_init_remote_repository

23
install.yml Executable file
View File

@ -0,0 +1,23 @@
---
- name: Setup apt pref
ansible.builtin.copy:
dest: /etc/apt/preferences.d/borgbackup.pref
owner: root
group: root
mode: '0644'
content: |
# Ansible managed
Package: borgbackup
Pin: release a={{ ansible_distribution_release|lower }}-backports
Pin-Priority: 990
when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
- name: Install main packages
ansible.builtin.package:
name: "{{ borg_packages }}"
- name: Install pip3 packages
ansible.builtin.pip:
name: "{{ borg_packages_pip }}"
state: latest
executable: pip3

24
main.yml Executable file
View File

@ -0,0 +1,24 @@
---
- name: assert variables
ansible.builtin.assert:
that: borg_encryption_passphrase is defined
- include_tasks: install.yml
tags:
- borg
- borg_install
- include_tasks: init.yml
tags:
- borg
- borg_init
- include_tasks: config.yml
tags:
- borg
- borg_config
- include_tasks: extra.yml
tags:
- borg
- borg_extra

2
meta/.galaxy_install_info Executable file
View File

@ -0,0 +1,2 @@
install_date: 'Sun 16 Jun 2024 04:56:29 PM '
version: ''

11
meta/main.yml Executable file
View File

@ -0,0 +1,11 @@
---
galaxy_info:
author: NoxInmortus
license: MIT
description: Role to deploy borgbackup and borgmatic configurations.
min_ansible_version: 2.7
platforms:
- name: Debian
galaxy_tags:
- backup
dependencies: []

137
s3-synchronize.sh Executable file
View File

@ -0,0 +1,137 @@
#!/usr/bin/env bash
set -euo pipefail
#=======================================#
#-- S3 SYNCHRONIZE SCRIPT --#
#-- Author : NoxInmortus (Alban E.G.) --#
#=======================================#
## {{{ Global variables
##------------------##
## Global Variables ##
##------------------##
SUBJECT="s3-synchronize"
VERSION="1.0.0 (18/03/2020)"
USAGE="Usage: ${0} -hv \n
-u : upload (local to remote) \n
-d : download (remote to local) \n
-l : local path to synchronize \n
-r : remote path to synchronize (s3 url format) \n
-c : s3cmd config file (mandatory)."
HELP="S3 synchronize script through s3cmd (both directions available). s3cmd binary required."
LOG="/var/log/${SUBJECT}.log"
DATE=$(date '+%F-%Hh')
## Global variables }}}
## {{{ Script variables
# SECONDS returns a count of the number of (whole) seconds the shell has been running.
startTime=${SECONDS}
# Used to check conflict
u_arg=false
d_arg=false
## Script variables }}}
## {{{ Option processing
##-------------------##
## Option processing ##
##-------------------##
# If there is no arguments display ${USAGE}
if [ $# == 0 ] ; then
echo -e ${USAGE}
exit 1;
fi
while getopts ":vhudl:r:c:" optname
do
case "${optname}" in
"v")
echo "Version ${VERSION}"
exit 0;
;;
"h")
echo -e ${HELP}
echo -e ${USAGE}
exit 0;
;;
"u")
sync_way="upload"
u_arg=true
;;
"d")
sync_way="download"
d_arg=true
;;
"l")
LOCAL=${OPTARG}
;;
"r")
REMOTE=${OPTARG}
;;
"c")
CONFIG_FILE=${OPTARG}
;;
"?")
echo "Unknown option ${OPTARG}"
exit 0;
;;
":")
echo "No argument value for option ${OPTARG}"
exit 0;
;;
*)
echo "Unknown error while processing options"
exit 0;
;;
esac
done
shift $((${OPTIND} - 1))
## Option processing }}}
# -----------------------------------------------------------------
# SCRIPT LOGIC GOES HERE
# -----------------------------------------------------------------
## Sanity Checks
if [[ -z ${LOCAL+x} ]]; then
echo "You need to define local directory (with -l option)."
exit 1
elif [[ -z ${REMOTE+x} ]]; then
echo "You need to define remote directory (with -r option)."
exit 1
elif [[ -z ${CONFIG_FILE+x} ]]; then
echo "You need to define s3cmd config file (with -c option)."
exit 1
fi
if [ ${u_arg} == ${d_arg} ]; then
echo "You cannot use -u (upload) and -d (download) options at the same time."
exit 1
fi
if [ ! -d ${LOCAL} ]; then
echo "Your local path is not a directory."
exit 1
fi
## Lockfile
(flock -x 200 || exit 1
if [ ${sync_way} == "upload" ]; then
s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error --delete-removed ${LOCAL} ${REMOTE}
elif [ ${sync_way} == "download" ]; then
s3cmd -c ${CONFIG_FILE} sync -v --stats --progress --stop-on-error ${REMOTE} ${LOCAL}
chmod 0700 ${LOCAL}
else
echo "Error with sync_way variable."
exit 1
fi
elapsedTime=$((${SECONDS} - ${startTime}))
echo "${SUBJECT} duration : $((${elapsedTime}/60)) min $((${elapsedTime}%60)) sec"
)200>/var/lock/${SUBJECT}.lock
exit 0

25
tasks/config.yml Executable file
View File

@ -0,0 +1,25 @@
---
- name: Create /etc/borgmatic directory
ansible.builtin.file:
state: 'directory'
path: '/etc/borgmatic'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0700'
- name: Copy borgmatic exclude file
ansible.builtin.template:
src: '{{ borg_exclude_template }}'
dest: '/etc/borgmatic/{{ borg_exclude_template|basename }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
- name: Copy borgmatic config file
ansible.builtin.template:
src: '{{ borg_conf_template }}'
dest: '/etc/borgmatic/{{ borg_conf_template|basename }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
validate: borgmatic config validate -c %s

76
tasks/extra.yml Executable file
View File

@ -0,0 +1,76 @@
---
- name: Set up borg cron job
ansible.builtin.cron:
name: 'Borgbackup'
cron_file: borgbackup
user: "{{ borg_user }}"
hour: '{{ borg_cron.hour }}'
minute: '{{ borg_cron.minute }}'
day: "{{ borg_cron.day }}"
weekday: "{{ borg_cron.weekday }}"
month: "{{ borg_cron.month }}"
job: >
PATH=$PATH:/usr/local/bin
nice -n {{ borg_cron_nice }}
ionice -c {{ borg_cron_ionice }}
borgmatic {{ borg_cron_action }} --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1
when: borg_cron_enable
- name: Set up borg prune cron job
ansible.builtin.cron:
name: 'Borgbackup Prune'
cron_file: borgbackup
user: "{{ borg_user }}"
hour: >-
{% if borg_cron.hour in range(0, 22) %}
{{ '%02d' | format(borg_cron.hour + 2) }}
{%- elif borg_cron.hour in [22, 23] -%}
{% if borg_cron.hour == 22 %}0{% else %}1{% endif %}
{%- endif -%}
minute: '{{ borg_cron.minute }}'
day: "{{ borg_cron.day }}"
weekday: "{{ borg_cron.weekday }}"
month: "{{ borg_cron.month }}"
job: >
PATH=$PATH:/usr/local/bin
nice -n {{ borg_cron_nice }}
ionice -c {{ borg_cron_ionice }}
borgmatic prune --log-file {{ borg_cron_log }} --log-file-verbosity 2 -c /etc/borgmatic/config.yaml >>{{ borg_cron_log }} 2>&1
when:
- borg_cron_enable
- borg_cron_action != 'prune'
- name: Create borg_cron_log file
ansible.builtin.file:
state: touch
modification_time: preserve
access_time: preserve
path: '{{ borg_cron_log }}'
owner: '{{ borg_user }}'
group: '{{ borg_user }}'
mode: '0600'
when: borg_cron_enable
- name: Copy s3-synchronize.sh scripts
ansible.builtin.copy:
src: files/s3-synchronize.sh
dest: /usr/local/bin/s3-synchronize.sh
mode: '0755'
when: borg_scripts
- name: Setup logrotate
ansible.builtin.copy:
dest: /etc/logrotate.d/borg
owner: root
group: root
mode: '0644'
content: |
{{ borg_cron_log }} {
daily
rotate 7
missingok
notifempty
compress
delaycompress
}
when: borg_logrotate

14
tasks/init.yml Executable file
View File

@ -0,0 +1,14 @@
---
- name: Init local borg repository
ansible.builtin.shell: 'BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_local_repository }} -e {{ borg_encryption_type }}'
args:
creates: "{{ borg_local_repository }}/data"
no_log: "{{ no_log|default('true') }}"
when: not borg_no_local_repository
- name: Init remote borg repository
ansible.builtin.shell: 'sudo -u {{ borg_user }} BORG_PASSPHRASE={{ borg_encryption_passphrase }} borg init {{ borg_remote_repository }} -e {{ borg_encryption_type }}'
no_log: "{{ no_log|default('true') }}"
when:
- borg_remote_repository is defined
- borg_init_remote_repository

24
tasks/install.yml Executable file
View File

@ -0,0 +1,24 @@
---
- name: Setup apt pref
ansible.builtin.copy:
dest: /etc/apt/preferences.d/borgbackup.pref
owner: root
group: root
mode: '0644'
content: |
# Ansible managed
Package: borgbackup
Pin: release a={{ ansible_distribution_release|lower }}-backports
Pin-Priority: 990
when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
- name: Install main packages
ansible.builtin.package:
name: "{{ borg_packages }}"
- name: Install pip3 packages
ansible.builtin.pip:
name: "{{ borg_packages_pip }}"
state: latest
executable: pip3
break_system_packages: true

24
tasks/main.yml Executable file
View File

@ -0,0 +1,24 @@
---
- name: assert variables
ansible.builtin.assert:
that: borg_encryption_passphrase is defined
- include_tasks: install.yml
tags:
- borg
- borg_install
- include_tasks: init.yml
tags:
- borg
- borg_init
- include_tasks: config.yml
tags:
- borg
- borg_config
- include_tasks: extra.yml
tags:
- borg
- borg_extra

50
templates/config.yaml Executable file
View File

@ -0,0 +1,50 @@
---
# ansible_managed /!\
location:
source_directories: [{% if borg_backup_dirs|length > 0 %}'{{ borg_backup_dirs|join("','") }}'{% endif %}]
repositories:
{%+ if not borg_no_local_repository %} - {{ borg_local_repository }}{% endif %}
{%+ if borg_remote_repository is defined %} - {{ borg_remote_repository }}{% endif %}
{%+ for key,value in borg_conf_location.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
storage:
{%+ for key,value in borg_conf_storage.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
retention:
{%+ for key,value in borg_conf_retention_policy.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
consistency:
{%+ for key,value in borg_conf_consistency.items()|sort %}
{{ key }}: {{ value }}
{% endfor %}
hooks:
umask: {{ borg_conf_umask }}
before_backup: [{% if borg_before_backup_commands|length > 0 %}'{{ borg_before_backup_commands|join("','") }}'{% endif %}]
after_backup: [{% if borg_after_backup_commands|length > 0 %}'{{ borg_after_backup_commands|join("','") }}'{% endif %}]
on_error: [{% if borg_failure_commands|length > 0 %}'{{ borg_failure_commands|join(",") }}'{% endif %}]
before_everything: [{% if borg_before_everything_commands|length > 0 %}'{{ borg_before_everything_commands|join(",") }}'{% endif %}]
after_everything: [{% if borg_after_everything_commands|length > 0 %}'{{ borg_after_everything_commands|join(",") }}'{% endif %}]
before_check: [{% if borg_before_check_commands|length > 0 %}'{{ borg_before_check_commands|join(",") }}'{% endif %}]
after_check: [{% if borg_after_check_commands|length > 0 %}'{{ borg_after_check_commands|join(",") }}'{% endif %}]
before_prune: [{% if borg_before_prune_commands|length > 0 %}'{{ borg_before_prune_commands|join(",") }}'{% endif %}]
after_prune: [{% if borg_after_prune_commands|length > 0 %}'{{ borg_after_prune_commands|join(",") }}'{% endif %}]
before_extract: [{% if borg_before_extract_commands|length > 0 %}'{{ borg_before_extract_commands|join(",") }}'{% endif %}]
after_extract: [{% if borg_after_extract_commands|length > 0 %}'{{ borg_after_extract_commands|join(",") }}'{% endif %}]
{% if borg_mysqldump is defined %}
mysql_databases:
{% for db in borg_mysqldump|default([]) %}
{% for key, value in db.items()|sort %}
{% if loop.first %}- {% else %} {% endif %}{{ key }}: {{ value }}
{% endfor %}
{% endfor %}
{% endif %}

5
templates/excludes Executable file
View File

@ -0,0 +1,5 @@
# ansible_managed /!\
{% for exclude in borg_excludes|default([]) + borg_excludes_default -%}
{{ exclude }}
{% endfor -%}

8
tests/.ansible-lint Executable file
View File

@ -0,0 +1,8 @@
parseable: true
use_default_rules: true
verbosity: 1
skip_list:
- role-name
- line-length
- command-instead-of-shell
- package-latest

11
tests/ansible.cfg Executable file
View File

@ -0,0 +1,11 @@
[defaults]
inventory = hosts
host_key_checking = false
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /tmp
roles_path = roles
timeout = 10
module_name = shell
retry_files_enabled = false
interpreter_python = /usr/bin/python3

1
tests/hosts Executable file
View File

@ -0,0 +1 @@
localhost ansible_connection=local

5
tests/playbook.yml Executable file
View File

@ -0,0 +1,5 @@
---
- hosts: localhost
gather_facts: true
roles:
- {role: default}