From ae270aeebd87769a2f5240aa427a7e6c87b2be30 Mon Sep 17 00:00:00 2001 From: Martin Kennedy Date: Sat, 27 Aug 2022 19:17:41 -0400 Subject: [PATCH 1/4] [v2] feat: lvm: Add lvm-based backup functionality This commit implements the needs of #75[^1]: it allows for the creation of atomic backups when the backup target is a file/dir whose fs rests on LVM. This ensures the snapshot will be atomic. By using a mount namespace, the LVM snapshot can be done in the same directory -- so existing LVM-based applications of ansible_role_restic can be migrated to this implementation without any discontinuity in what appears to be backed up. This combination of LVM's snapshotting layer and mount namespaces comes with some caveats: - you cannot backup / due to namespace issues - subdirs with a separate fs won't be correctly detected - not all filesystems are happy about LVM snapshots -- btrfs, e.g. - LVM snapshots come with a performance penalty when active - fstrim and LVM snapshots don't like each other whatsoever [^1]: https://github.com/roles-ansible/ansible_role_restic/issues/75 -- Changes in v2: - Use `findmnt -v` to find snapshot when cleaning up - Check for _snap before `lvremove -y` Signed-off-by: Martin Kennedy --- templates/restic.service.j2 | 7 ++++ templates/restic_script_Linux.j2 | 70 +++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/templates/restic.service.j2 b/templates/restic.service.j2 index 9ad9de4..924381b 100644 --- a/templates/restic.service.j2 +++ b/templates/restic.service.j2 @@ -1,8 +1,15 @@ [Unit] Description=Backup {{ item.name }} using restic +{% if item.lvm is defined %} +Conflicts=fstrim.service +After=fstrim.timer +{% endif %} [Service] Type=oneshot +{% if item.lvm is defined %} +PrivateMounts=on +{% endif %} ExecStart={{ restic_script_dir }}/backup-{{ item.name }}.sh TimeoutStartSec=0 Environment="CRON=true" diff --git a/templates/restic_script_Linux.j2 b/templates/restic_script_Linux.j2 index e699a8b..178a1cd 100644 --- a/templates/restic_script_Linux.j2 +++ b/templates/restic_script_Linux.j2 @@ -62,6 +62,67 @@ export B2_ACCOUNT_KEY={{ restic_repos[item.repo].b2_account_key }} BACKUP_SOURCE={{ item.src }} {% endif %} +{% if item.lvm is defined %} +# Set up functions for LVM. + +function mount_opt_map { + mount_type="$1" + case "$mount_type" in + xfs) + echo "noatime,nouuid" + ;; + ext4) + echo "noatime" + ;; + *) + echo "noatime" + esac +} + +function prepare_vol { + local path="$1" + [ -d "$path" ] || path="$(dirname "$path")" + + # TODO: path cannot be /, + ## nor can it be where restic is + { + local source="$(findmnt -J -T ${path} | jq -r '.filesystems[0].source')" + local target="$(findmnt -J -T ${path} | jq -r '.filesystems[0].target')" + subdir=${path##$target} + echo "Creating snapshot ..." + lvcreate -y -L "${size:-10G}" -s -n "${source}_snap" "${source}" + + local tmpdir="$(mktemp -d)" + local fs="$(lsblk -J --fs "$source" | jq -r '.blockdevices[0]|.fstype')" + echo "Identified fstype: $fs; using opts $(mount_opt_map "$fs") ..." + mount -t "$fs" \ + -o "$(mount_opt_map "$fs")" \ + --make-private \ + -m \ + "${source}_snap" "${tmpdir}" + + mount -m --bind --make-private "${tmpdir}/${subdir}" "${path}" + } +} + +function cleanup_vol { + local path="$1" + [ -d "$path" ] || path="$(dirname "$path")" + + { + local source="$(findmnt -v -f -J -T ${path} | jq -r '.filesystems[0].source')" + echo "Cleaning up mount ..." + umount "${path}" + + echo "Cleaning up snapshot ..." + if ! grep -q '_snap$' <<< $source; then + echo "Snapshot for ${path} could not be found (found: ${source}). Exiting!" && return 1; + fi + umount "${source}" + lvremove -y "${source}"; + } +} +{% endif %} set -uxo pipefail {# @@ -150,10 +211,14 @@ fi {% if item.exclude is defined %}{{ exclude(item.exclude) }}{% endif %} \ $@ \ {% else %} +{ + {% if item.lvm is defined %}prepare_vol $BACKUP_SOURCE &&{% endif %} {{ restic_install_path }}/restic backup $BACKUP_SOURCE $MODE_TAG \ {{ tags(item.tags) }} \ {% if item.exclude is defined %}{{ exclude(item.exclude) }}{% endif %} \ $@ \ + {% if item.lvm is defined %}&& cleanup_vol $BACKUP_SOURCE{% endif %}; +} \ {% endif %} {{ backup_output_log }} if [[ $? -eq 0 ]] then @@ -166,7 +231,10 @@ else {{ ' ' }}We tried to backup '{{ item.src }}'. {%- endif -%} {{ ' ' }}Please repair the restic-{{ item.name | replace(' ', '') }} job." - {% endif %} +{% if item.lvm is defined %} + cleanup_vol $BACKUP_SOURCE +{% endif %} +{% endif %} fi From 4cf66ebec4368531be8dfc36cae6d88f5f3b5ade Mon Sep 17 00:00:00 2001 From: Martin Kennedy Date: Sun, 28 Aug 2022 16:19:51 -0400 Subject: [PATCH 2/4] fix: More tweaks to get snapshot-finding to work Grr! this is really frustrating. We need more "tracking" for the snapshot we created. --- templates/restic_script_Linux.j2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/templates/restic_script_Linux.j2 b/templates/restic_script_Linux.j2 index 178a1cd..3177e5d 100644 --- a/templates/restic_script_Linux.j2 +++ b/templates/restic_script_Linux.j2 @@ -110,14 +110,16 @@ function cleanup_vol { [ -d "$path" ] || path="$(dirname "$path")" { - local source="$(findmnt -v -f -J -T ${path} | jq -r '.filesystems[0].source')" + local source="$(findmnt -v -J -T ${path} | jq -r '.filesystems[]|.source' | grep '_snap$')" + + if ! grep -q '_snap$' <<< $source; then + echo "Snapshot for ${path} could not be found (found: ${source}). Exiting!" && return 1; + fi + echo "Cleaning up mount ..." umount "${path}" echo "Cleaning up snapshot ..." - if ! grep -q '_snap$' <<< $source; then - echo "Snapshot for ${path} could not be found (found: ${source}). Exiting!" && return 1; - fi umount "${source}" lvremove -y "${source}"; } From de3d35d4ece5184f89ce4124823d122274cc612f Mon Sep 17 00:00:00 2001 From: Martin Kennedy Date: Tue, 7 Mar 2023 16:24:30 -0500 Subject: [PATCH 3/4] fix: lvm: Allow / to be backed up as LVM This commit causes an LVM backup target of / to be treated as /rootfs instead. Note that this will conflict with a path called /rootfs if we *do* try to back that up. Also, while we're at it, drop the -m option from mount. Not all systems have this option. --- templates/restic_script_Linux.j2 | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/templates/restic_script_Linux.j2 b/templates/restic_script_Linux.j2 index 3177e5d..d274e36 100644 --- a/templates/restic_script_Linux.j2 +++ b/templates/restic_script_Linux.j2 @@ -82,9 +82,13 @@ function mount_opt_map { function prepare_vol { local path="$1" [ -d "$path" ] || path="$(dirname "$path")" + if [ "$path" == '/' ] ; then + mkdir -p /rootfs; + newpath='/rootfs'; + else + newpath="$path"; + fi - # TODO: path cannot be /, - ## nor can it be where restic is { local source="$(findmnt -J -T ${path} | jq -r '.filesystems[0].source')" local target="$(findmnt -J -T ${path} | jq -r '.filesystems[0].target')" @@ -98,26 +102,30 @@ function prepare_vol { mount -t "$fs" \ -o "$(mount_opt_map "$fs")" \ --make-private \ - -m \ "${source}_snap" "${tmpdir}" - mount -m --bind --make-private "${tmpdir}/${subdir}" "${path}" + mount --bind --make-private "${tmpdir}/${subdir}" "${newpath}" } } function cleanup_vol { local path="$1" [ -d "$path" ] || path="$(dirname "$path")" + if [ "$path" == '/' ] ; then + newpath='/rootfs'; + else + newpath="$path"; + fi { - local source="$(findmnt -v -J -T ${path} | jq -r '.filesystems[]|.source' | grep '_snap$')" + local source="$(findmnt -v -J -T "${newpath}" | jq -r '.filesystems[]|.source' | grep '_snap$')" if ! grep -q '_snap$' <<< $source; then echo "Snapshot for ${path} could not be found (found: ${source}). Exiting!" && return 1; fi echo "Cleaning up mount ..." - umount "${path}" + umount "${newpath}" echo "Cleaning up snapshot ..." umount "${source}" @@ -215,7 +223,7 @@ fi {% else %} { {% if item.lvm is defined %}prepare_vol $BACKUP_SOURCE &&{% endif %} - {{ restic_install_path }}/restic backup $BACKUP_SOURCE $MODE_TAG \ + {{ restic_install_path }}/restic backup {% if item.lvm is defined and item.src == '/' %}/rootfs{% endif %}$BACKUP_SOURCE $MODE_TAG \ {{ tags(item.tags) }} \ {% if item.exclude is defined %}{{ exclude(item.exclude) }}{% endif %} \ $@ \ From 6459eaa161e21d32065ec99d681161c03d63e909 Mon Sep 17 00:00:00 2001 From: Martin Kennedy Date: Tue, 7 Mar 2023 16:26:36 -0500 Subject: [PATCH 4/4] fix: lvm: Handle cleanup after unreadable files Before this commit, since restic backup exits with status 3 if it cannot read one or more files, the LVM snapshot wasn't being cleaned up. Now, specially handle the 3 exit status; also, unequivocally perform the LVM cleanup when finished. --- templates/restic_script_Linux.j2 | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/templates/restic_script_Linux.j2 b/templates/restic_script_Linux.j2 index d274e36..3647f8a 100644 --- a/templates/restic_script_Linux.j2 +++ b/templates/restic_script_Linux.j2 @@ -226,27 +226,31 @@ fi {{ restic_install_path }}/restic backup {% if item.lvm is defined and item.src == '/' %}/rootfs{% endif %}$BACKUP_SOURCE $MODE_TAG \ {{ tags(item.tags) }} \ {% if item.exclude is defined %}{{ exclude(item.exclude) }}{% endif %} \ - $@ \ - {% if item.lvm is defined %}&& cleanup_vol $BACKUP_SOURCE{% endif %}; + $@ } \ {% endif %} {{ backup_output_log }} -if [[ $? -eq 0 ]] -then - echo "$(date -u '+%Y-%m-%d %H:%M:%S') OK" {{ backup_result_log }} -else - echo "$(date -u '+%Y-%m-%d %H:%M:%S') ERROR" {{ backup_result_log }} + +case $? in + 0) + echo "$(date -u '+%Y-%m-%d %H:%M:%S') OK" {{ backup_result_log }} + ;; + 3) + echo "$(date -u '+%Y-%m-%d %H:%M:%S') WARNING" {{ backup_result_log }} + ;; + *) + echo "$(date -u '+%Y-%m-%d %H:%M:%S') ERROR" {{ backup_result_log }} {% if item.mail_on_error is defined and item.mail_on_error == true %} mail -s "restic backup failed on {{ ansible_hostname }}" {{ item.mail_address }} <<< "Something went wrong while running restic backup script running at {{ ansible_hostname }} at $(date -u '+%Y-%m-%d %H:%M:%S'). {%- if item.src is defined -%} {{ ' ' }}We tried to backup '{{ item.src }}'. {%- endif -%} {{ ' ' }}Please repair the restic-{{ item.name | replace(' ', '') }} job." +{% endif %} +esac + {% if item.lvm is defined %} cleanup_vol $BACKUP_SOURCE {% endif %} -{% endif %} -fi - {#