From 7361da8348faba86ba02d595aece81205aae0d2a Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 19 Apr 2023 13:36:19 +0200 Subject: [PATCH] removed backup links and versions folder --- .../files/backups-cleanup.py | 12 ++-- .../files/pull-remote-backup.sh | 70 ++++++++----------- .../ssh-wrapper.sh} | 13 ++-- .../server_native-user-backup/tasks/main.yml | 8 +-- 4 files changed, 47 insertions(+), 56 deletions(-) rename roles/server_native-user-backup/{templates/ssh-wrapper.sh.j2 => files/ssh-wrapper.sh} (64%) diff --git a/roles/server_native-backups-cleanup/files/backups-cleanup.py b/roles/server_native-backups-cleanup/files/backups-cleanup.py index 45e9e70a..8fdf05fa 100644 --- a/roles/server_native-backups-cleanup/files/backups-cleanup.py +++ b/roles/server_native-backups-cleanup/files/backups-cleanup.py @@ -13,11 +13,13 @@ deleted = True while psutil.disk_usage(args.backups_folder_path).percent > args.maximum_backup_size_percent and deleted: deleted = False print("%d %% of disk %s are used. Freeing space..." % (psutil.disk_usage(args.backups_folder_path).percent,args.backups_folder_path)) - for primary_directory in os.listdir(args.backups_folder_path): - primary_directory = os.path.join(args.backups_folder_path, primary_directory) - for application_directory in os.listdir(primary_directory): - application_directory = os.path.join(primary_directory, application_directory) - versions_directory = os.path.join(application_directory, "versions/") + for host_backup_directory_name in os.listdir(args.backups_folder_path): + host_backup_directory_path = os.path.join(args.backups_folder_path, host_backup_directory_name) + for application_directory in os.listdir(host_backup_directory_path): + + # The directory which contains all backup versions of the application + versions_directory = os.path.join(host_backup_directory_path, application_directory) + versions = os.listdir(versions_directory) versions.sort(reverse=False) if len(versions) >= 1: diff --git a/roles/server_native-pull-primary-backups/files/pull-remote-backup.sh b/roles/server_native-pull-primary-backups/files/pull-remote-backup.sh index 696527c1..ba3dfd55 100644 --- a/roles/server_native-pull-primary-backups/files/pull-remote-backup.sh +++ b/roles/server_native-pull-primary-backups/files/pull-remote-backup.sh @@ -1,63 +1,53 @@ #!/bin/bash # @param $1 hostname from which backup should be pulled -echo "pulling backups from: $1" +echo "pulling backups from: $1" && # error counter -errors=0 +errors=0 && -echo "loading meta data..." +echo "loading meta data..." && -remote_host="backup@$1" -echo "host address: $remote_host" +remote_host="backup@$1" && +echo "host address: $remote_host" && -remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" || exit 1 -echo "remote machine id: $remote_machine_id" +remote_machine_id="$( (ssh "$remote_host" sha256sum /etc/machine-id) | head -c 64 )" && +echo "remote machine id: $remote_machine_id" && -general_backup_machine_dir="/Backups/$remote_machine_id/" -echo "backup dir: $general_backup_machine_dir" +general_backup_machine_dir="/Backups/$remote_machine_id/" && +echo "backup dir: $general_backup_machine_dir" && -remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" || exit 1 -echo "backuptypes: $remote_backup_types" +remote_backup_types="$(ssh "$remote_host" "find $general_backup_machine_dir -maxdepth 1 -type d -execdir basename {} ;")" && +echo "backup types: $remote_backup_types" || exit 1 for backup_type in $remote_backup_types; do if [ "$backup_type" != "$remote_machine_id" ]; then - general_backup_type_dir="$general_backup_machine_dir""$backup_type/" - # folder which contains versions - general_versions_dir="$general_backup_type_dir""versions/" - # link name of last backup - general_latest_version_link="$general_backup_type_dir""latest" + echo "backup type: $backup_type" && - # this folder contains the last backup - local_latest_version_dir="$general_versions_dir$(date '+%Y%m%d%H%M%S')/" - # this is the link name of the previous version - local_previous_version_link="$general_backup_type_dir""previous" + general_backup_type_dir="$general_backup_machine_dir""$backup_type/" && + general_versions_dir="$general_backup_type_dir" && + local_previous_version_dir="$(ls -d $general_versions_dir* | tail -1)" && + echo "last local backup: $local_previous_version_dir" && - #identifiy previous version - local_versions=( $(basename -a "$general_versions_dir"*/ | sort) )|| exit 1 - local_last_version="${local_versions[-1]}" || exit 1 - local_previous_version_dir="$general_versions_dir""$local_last_version/" + remote_backup_versions="$(ssh "$remote_host" ls -d "$general_backup_type_dir"\*)" && + echo "remote backup versions: $remote_backup_versions" && - # source path of the backup files: - remote_last_version_dir="$(ssh "$remote_host" readlink -f $general_latest_version_link)" - echo "last remote backup: $remote_last_version_dir" - remote_source_path="$remote_host:$remote_last_version_dir/" - echo "source path: $remote_source_path" - # create working folders: - mkdir -vp "$local_latest_version_dir" + remote_last_backup_dir=$(echo "$remote_backup_versions" | tail -1) && + echo "last remote backup: $remote_last_backup_dir" && - # delete links - rm -v "$general_latest_version_link" - rm -v "$local_previous_version_link" + remote_source_path="$remote_host:$remote_last_backup_dir/" && + echo "source path: $remote_source_path" && - # create links - ln -vs "$local_latest_version_dir" "$general_latest_version_link" || exit 1 - ln -vs "$local_previous_version_dir" "$local_previous_version_link" || exit 1 + local_backup_destination_path=$remote_last_backup_dir && + echo "backup destination: $local_backup_destination_path" && - # do backup: - rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_link'" "'$remote_source_path'" "'$general_latest_version_link'"' - echo "executing: $rsync_command" + echo "creating local backup destination folder..." && + mkdir -vp "$local_backup_destination_path" && + + echo "starting backup..." && + rsync_command='rsync -abP --delete --delete-excluded --rsync-path="sudo rsync" --link-dest="'$local_previous_version_dir'" "'$remote_source_path'" "'$local_backup_destination_path'"' && + echo "executing: $rsync_command" && eval "$rsync_command" || ((errors+=1)); fi done diff --git a/roles/server_native-user-backup/templates/ssh-wrapper.sh.j2 b/roles/server_native-user-backup/files/ssh-wrapper.sh similarity index 64% rename from roles/server_native-user-backup/templates/ssh-wrapper.sh.j2 rename to roles/server_native-user-backup/files/ssh-wrapper.sh index 2eb92b9d..6c8ae350 100644 --- a/roles/server_native-user-backup/templates/ssh-wrapper.sh.j2 +++ b/roles/server_native-user-backup/files/ssh-wrapper.sh @@ -8,19 +8,22 @@ fi # define executable commands get_hashed_machine_id="sha256sum /etc/machine-id"; -get_backup_types="find /Backups/{{hashed_machine_id.stdout}}/ -maxdepth 1 -type d -execdir basename {} ;"; +hashed_machine_id="$($get_hashed_machine_id | head -c 64)" +get_backup_types="find /Backups/$hashed_machine_id/ -maxdepth 1 -type d -execdir basename {} ;"; + # @todo This configuration is not scalable yet. If other backup services then docker-volume-backup are integrated, this logic needs to be optimized -get_static_last_version_dir="readlink -f /Backups/{{hashed_machine_id.stdout}}/docker-volume-backup/latest" -rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $($get_static_last_version_dir)/" +get_version_directories="ls -d /Backups/$hashed_machine_id/docker-volume-backup/*" +last_version_directory="$($get_version_directories | tail -1)" +rsync_command="sudo rsync --server --sender -blogDtpre.iLsfxCIvu . $last_version_directory/" # filter commands case "$SSH_ORIGINAL_COMMAND" in "$get_hashed_machine_id") $get_hashed_machine_id ;; - "$get_static_last_version_dir") - $get_static_last_version_dir + "$get_version_directories") + $get_version_directories ;; "$get_backup_types") $get_backup_types diff --git a/roles/server_native-user-backup/tasks/main.yml b/roles/server_native-user-backup/tasks/main.yml index 81876818..53a6caa2 100644 --- a/roles/server_native-user-backup/tasks/main.yml +++ b/roles/server_native-user-backup/tasks/main.yml @@ -11,10 +11,6 @@ group: backup mode: '0700' -- name: register hashed_machine_id - shell: sha256sum /etc/machine-id | head -c 64 - register: hashed_machine_id - - name: create /home/backup/.ssh/authorized_keys template: src: "authorized_keys.j2" @@ -24,8 +20,8 @@ mode: '0644' - name: create /home/backup/ssh-wrapper.sh - template: - src: "ssh-wrapper.sh.j2" + copy: + src: "ssh-wrapper.sh" dest: /home/backup/ssh-wrapper.sh owner: backup group: backup